Implement integration tests for remote template and workflow urls.

integration_tests/loader/basic.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+  - template: workflow/match-2.yaml

integration_tests/loader/condition-matched.yaml

@@ -0,0 +1,11 @@
+id: condition-matched-workflow
+
+info:
+  name: Condition Matched Workflow
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/match-2.yaml

integration_tests/loader/get-headers.yaml

@@ -0,0 +1,17 @@
+id: basic-get-headers
+
+info:
+  name: Basic GET Headers Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      test: nuclei
+    matchers:
+      - type: word
+        words:
+          - "This is test headers matcher text"

integration_tests/loader/get.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

integration_tests/loader/template-list.yaml

@@ -0,0 +1,2 @@
+loader/get.yaml
+loader/get-headers.yaml

integration_tests/loader/workflow-list.yaml

@@ -0,0 +1,2 @@
+loader/basic.yaml
+loader/condition-matched.yaml

v2/cmd/integration-test/integration-test.go

@@ -26,6 +26,7 @@ func main() {
 		"network":  networkTestcases,
 		"dns":      dnsTestCases,
 		"workflow": workflowTestcases,
+		"loader":   loaderTestcases,
 	}
 	for proto, tests := range protocolTests {
 		if protocol == "" || protocol == proto {

v2/cmd/integration-test/loader.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"fmt"
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+)
+
+var loaderTestcases = map[string]testutils.TestCase{
+	"loader/template-list.yaml":             &remoteTemplateList{},
+	"loader/workflow-list.yaml":             &remoteWorkflowList{},
+	"loader/nonexistent-template-list.yaml": &nonExistentTemplateList{},
+	"loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{},
+}
+
+type remoteTemplateList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteTemplateList) Execute(templateList string) error {
+	router := httprouter.New()
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+
+	router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, err := os.ReadFile(templateList)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+		w.Write(file)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	if err != nil {
+		return err
+	}
+	if len(results) != 2 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type remoteWorkflowList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *remoteWorkflowList) Execute(workflowList string) error {
+	router := httprouter.New()
+
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+		if strings.EqualFold(r.Header.Get("test"), "nuclei") {
+			fmt.Fprintf(w, "This is test headers matcher text")
+		}
+	}))
+
+	router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, err := os.ReadFile(workflowList)
+		if err != nil {
+			w.WriteHeader(500)
+		}
+		w.Write(file)
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list")
+	if err != nil {
+		return err
+	}
+	if len(results) != 3 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type nonExistentTemplateList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error {
+	router := httprouter.New()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	if err == nil {
+		return fmt.Errorf("expected error for nonexisting workflow url")
+	}
+
+	return nil
+}
+
+type nonExistentWorkflowList struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error {
+	router := httprouter.New()
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	if err == nil {
+		return fmt.Errorf("expected error for nonexisting workflow url")
+	}
+
+	return nil
+}

v2/internal/testutils/integration.go

@@ -12,15 +12,15 @@ import (
 
 // RunNucleiTemplateAndGetResults returns a list of results for a template
 func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	return runNucleiAndGetResults(true, template, url, debug, extra...)
+	return RunNucleiAndGetResults(true, template, url, debug, extra...)
 }
 
 // RunNucleiWorkflowAndGetResults returns a list of results for a workflow
 func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) {
-	return runNucleiAndGetResults(false, template, url, debug, extra...)
+	return RunNucleiAndGetResults(false, template, url, debug, extra...)
 }
 
-func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
+func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) {
 	var templateOrWorkflowFlag string
 	if isTemplate {
 		templateOrWorkflowFlag = "-t"
@@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e
 		templateOrWorkflowFlag = "-w"
 	}
 
-	cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent")
+	return RunNucleiBareArgsAndGetResults(debug, append([]string{
+		templateOrWorkflowFlag,
+		template,
+		"-target",
+		url,
+	}, extra...)...)
+}
+
+func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) {
+	cmd := exec.Command("./nuclei")
 	if debug {
-		cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug")
+		cmd.Args = append(cmd.Args, "-debug")
 		cmd.Stderr = os.Stderr
 		fmt.Println(cmd.String())
+	} else {
+		cmd.Args = append(cmd.Args, "-silent")
 	}
 	cmd.Args = append(cmd.Args, extra...)
 	data, err := cmd.Output()