.

v2/pkg/protocols/file/operators.go

@@ -1,14 +1,10 @@
 package file
 
 import (
-	"time"
-
-	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -65,26 +61,12 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 	return itemStr, true
 }
 
-type fileStatus struct {
-	results         []*operators.Result
-	raw             string
-	inputFilePath   string
-	matchedFileName string
-	lines           int
-	words           int
-	bytes           int
-}
-
 // responseToDSLMap converts a file chunk elaboration to a map for use in DSL matching
-func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent {
+func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent {
 	return output.InternalEvent{
-		"results":       state.results,
+		"path":          inputFilePath,
-		"path":          state.inputFilePath,
+		"matched":       matchedFileName,
-		"matched":       state.matchedFileName,
+		"raw":           raw,
-		"raw":           state.raw,
-		"lines":         state.lines,
-		"words":         state.words,
-		"bytes":         state.bytes,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,
 		"template-info": request.options.TemplateInfo,
@@ -93,27 +75,17 @@ func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent
 }
 
 // MakeResultEvent creates a result event from internal wrapped event
+// Deprecated: unused in stream mode, must be present for interface compatibility
 func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
-	return protocols.MakeDefaultResultEvent(request, wrapped)
+	panic("unused")
 }
 
 func (request *Request) GetCompiledOperators() []*operators.Operators {
 	return []*operators.Operators{request.CompiledOperators}
 }
 
+// MakeResultEventItem
+// Deprecated: unused in stream mode, must be present for interface compatibility
 func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
-	data := &output.ResultEvent{
+	panic("unused")
-		MatcherStatus:    true,
-		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
-		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
-		Info:             wrapped.InternalEvent["template-info"].(model.Info),
-		Type:             types.ToString(wrapped.InternalEvent["type"]),
-		Path:             types.ToString(wrapped.InternalEvent["path"]),
-		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
-		Host:             types.ToString(wrapped.InternalEvent["host"]),
-		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
-		Response:         types.ToString(wrapped.InternalEvent["raw"]),
-		Timestamp:        time.Now(),
-	}
-	return data
 }

v2/pkg/protocols/file/operators_test.go

@@ -1,6 +1,8 @@
 package file
 
 import (
+	"log"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -34,8 +36,8 @@ func TestResponseToDSLMap(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 }
 
@@ -59,8 +61,8 @@ func TestFileOperatorMatch(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("valid", func(t *testing.T) {
@@ -108,8 +110,8 @@ func TestFileOperatorMatch(t *testing.T) {
 
 	t.Run("caseInsensitive", func(t *testing.T) {
 		resp := "TEST-DATA\r\n1.1.1.1\r\n"
-		event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+		event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-		require.Len(t, event, 11, "could not get correct number of items in dsl map")
+		require.Len(t, event, 7, "could not get correct number of items in dsl map")
 		require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 		matcher := &matchers.Matcher{
@@ -147,8 +149,8 @@ func TestFileOperatorExtract(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("extract", func(t *testing.T) {
@@ -217,6 +219,7 @@ func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.
 	}
 
 	finalEvent := testFileMakeResult(t, matcher, matcherCondition, true)
+	log.Fatalf("%+v\n%+v\n", expectedValues, finalEvent.Results[0])
 	for matcherName, matchedValues := range expectedValues {
 		var matchesOne = false
 		for i := 0; i <= len(expectedValue); i++ {
@@ -262,24 +265,16 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 
-	matchedFileName := "test.txt"
 	fileContent := "test-data\r\n1.1.1.1\r\n"
+	matchedFileName := "test.txt"
+	input := "/tmp"
 
-	event := request.responseToDSLMap(&fileStatus{raw: fileContent, inputFilePath: "/tmp", matchedFileName: matchedFileName})
+	fileMatches := request.collectMatches(strings.NewReader(fileContent), input, matchedFileName, "")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	event := request.buildEvent(input, matchedFileName, fileMatches)
-	require.Equal(t, fileContent, event["raw"], "could not get correct resp")
 
-	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
+	resultEvent := event.Results[0]
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug)
-		if ok && result != nil {
-			finalEvent.OperatorsResult = result
-			finalEvent.Results = request.MakeResultEvent(finalEvent)
-		}
-	}
-	resultEvent := finalEvent.Results[0]
 	require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
 	require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value")
 
-	return finalEvent
+	return event
 }

v2/pkg/protocols/file/request.go

@@ -65,17 +65,34 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 			if stat.Size() >= request.maxSize {
 				gologger.Verbose().Msgf("Limiting %s processed data to %s bytes: exceeded max size\n", filePath, units.HumanSize(float64(request.maxSize)))
 			}
-			totalBytes := units.BytesSize(float64(stat.Size()))
+
 			fileReader := io.LimitReader(file, request.maxSize)
+			fileMatches := request.collectMatches(fileReader, input, filePath, units.BytesSize(float64(stat.Size())))
+
+			// build event structure to interface with internal logic
+			event := request.buildEvent(input, filePath, fileMatches)
+			dumpResponse(event, request.options, fileMatches, filePath)
+			callback(event)
+			request.options.Progress.IncrementRequests()
+		}(data)
+	})
+	wg.Wait()
+	if err != nil {
+		request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
+		request.options.Progress.IncrementFailedRequestsBy(1)
+		return errors.Wrap(err, "could not send file request")
+	}
+	return nil
+}
+
+func (request *Request) collectMatches(reader io.Reader, input, filePath, totalBytes string) []FileMatch {
 	var bytesCount, linesCount, wordsCount int
 	isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
-			scanner := bufio.NewScanner(fileReader)
+	scanner := bufio.NewScanner(reader)
 	buffer := []byte{}
 	scanner.Buffer(buffer, int(chunkSize))
 
 	var fileMatches []FileMatch
-			exprLines := make(map[string][]int)
-			exprBytes := make(map[string][]int)
 	for scanner.Scan() {
 		lineContent := scanner.Text()
 		n := len(lineContent)
@@ -85,15 +102,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 		processedBytes := units.BytesSize(float64(currentBytes))
 
 		gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytes)
-				dslMap := request.responseToDSLMap(&fileStatus{
+		dslMap := request.responseToDSLMap(lineContent, input, filePath)
-					raw:             lineContent,
-					inputFilePath:   input,
-					matchedFileName: filePath,
-					lines:           linesCount,
-					words:           wordsCount,
-					bytes:           bytesCount,
-				})
-
 		if parts, ok := request.CompiledOperators.Execute(dslMap, request.Match, request.Extract, isResponseDebug); parts != nil && ok {
 			if parts.Extracts != nil {
 				for expr, extracts := range parts.Extracts {
@@ -129,14 +138,14 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 		linesCount += currentLinesCount
 		wordsCount += strings.Count(lineContent, " ")
 		bytesCount = currentBytes
-
 	}
+	return fileMatches
+}
 
-			// build event structure to interface with internal logic
+func (request *Request) buildEvent(input, filePath string, fileMatches []FileMatch) *output.InternalWrappedEvent {
-			internalEvent := request.responseToDSLMap(&fileStatus{
+	exprLines := make(map[string][]int)
-				inputFilePath:   input,
+	exprBytes := make(map[string][]int)
-				matchedFileName: filePath,
+	internalEvent := request.responseToDSLMap("", input, filePath)
-			})
 	operatorResult := &operators.Result{}
 	for _, fileMatch := range fileMatches {
 		operatorResult.Matched = operatorResult.Matched || fileMatch.Match
@@ -180,7 +189,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 			Info:             internalEvent["template-info"].(model.Info),
 			Type:             types.ToString(internalEvent["type"]),
 			Path:             types.ToString(internalEvent["path"]),
-					Matched:          types.ToString(internalEvent["path"]),
+			Matched:          types.ToString(internalEvent["matched"]),
 			Host:             types.ToString(internalEvent["host"]),
 			ExtractedResults: items,
 			// Response:         types.ToString(wrapped.InternalEvent["raw"]),
@@ -207,23 +216,11 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 		})
 	}
 
-			event := &output.InternalWrappedEvent{
+	return &output.InternalWrappedEvent{
 		InternalEvent:   internalEvent,
 		Results:         results,
 		OperatorsResult: operatorResult,
 	}
-			dumpResponse(event, request.options, fileMatches, filePath)
-			callback(event)
-			request.options.Progress.IncrementRequests()
-		}(data)
-	})
-	wg.Wait()
-	if err != nil {
-		request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err)
-		request.options.Progress.IncrementFailedRequestsBy(1)
-		return errors.Wrap(err, "could not send file request")
-	}
-	return nil
 }
 
 func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, filematches []FileMatch, filePath string) {