.

v2/pkg/protocols/file/operators.go

@@ -1,14 +1,10 @@
 package file
 
 import (
-	"time"
-
-	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 
@@ -65,26 +61,12 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 	return itemStr, true
 }
 
-type fileStatus struct {
-	results         []*operators.Result
-	raw             string
-	inputFilePath   string
-	matchedFileName string
-	lines           int
-	words           int
-	bytes           int
-}
-
 // responseToDSLMap converts a file chunk elaboration to a map for use in DSL matching
-func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent {
+func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent {
 	return output.InternalEvent{
-		"results":       state.results,
+		"path":          inputFilePath,
-		"path":          state.inputFilePath,
+		"matched":       matchedFileName,
-		"matched":       state.matchedFileName,
+		"raw":           raw,
-		"raw":           state.raw,
-		"lines":         state.lines,
-		"words":         state.words,
-		"bytes":         state.bytes,
 		"type":          request.Type().String(),
 		"template-id":   request.options.TemplateID,
 		"template-info": request.options.TemplateInfo,
@@ -93,27 +75,17 @@ func (request *Request) responseToDSLMap(state *fileStatus) output.InternalEvent
 }
 
 // MakeResultEvent creates a result event from internal wrapped event
+// Deprecated: unused in stream mode, must be present for interface compatibility
 func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
-	return protocols.MakeDefaultResultEvent(request, wrapped)
+	panic("unused")
 }
 
 func (request *Request) GetCompiledOperators() []*operators.Operators {
 	return []*operators.Operators{request.CompiledOperators}
 }
 
+// MakeResultEventItem
+// Deprecated: unused in stream mode, must be present for interface compatibility
 func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
-	data := &output.ResultEvent{
+	panic("unused")
-		MatcherStatus:    true,
-		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
-		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
-		Info:             wrapped.InternalEvent["template-info"].(model.Info),
-		Type:             types.ToString(wrapped.InternalEvent["type"]),
-		Path:             types.ToString(wrapped.InternalEvent["path"]),
-		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
-		Host:             types.ToString(wrapped.InternalEvent["host"]),
-		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
-		Response:         types.ToString(wrapped.InternalEvent["raw"]),
-		Timestamp:        time.Now(),
-	}
-	return data
 }

v2/pkg/protocols/file/operators_test.go

@@ -1,6 +1,8 @@
 package file
 
 import (
+	"log"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -34,8 +36,8 @@ func TestResponseToDSLMap(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 }
 
@@ -59,8 +61,8 @@ func TestFileOperatorMatch(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("valid", func(t *testing.T) {
@@ -108,8 +110,8 @@ func TestFileOperatorMatch(t *testing.T) {
 
 	t.Run("caseInsensitive", func(t *testing.T) {
 		resp := "TEST-DATA\r\n1.1.1.1\r\n"
-		event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+		event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-		require.Len(t, event, 11, "could not get correct number of items in dsl map")
+		require.Len(t, event, 7, "could not get correct number of items in dsl map")
 		require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 		matcher := &matchers.Matcher{
@@ -147,8 +149,8 @@ func TestFileOperatorExtract(t *testing.T) {
 	require.Nil(t, err, "could not compile file request")
 
 	resp := "test-data\r\n1.1.1.1\r\n"
-	event := request.responseToDSLMap(&fileStatus{raw: resp, inputFilePath: "one.one.one.one", matchedFileName: "one.one.one.one"})
+	event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["raw"], "could not get correct resp")
 
 	t.Run("extract", func(t *testing.T) {
@@ -217,6 +219,7 @@ func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.
 	}
 
 	finalEvent := testFileMakeResult(t, matcher, matcherCondition, true)
+	log.Fatalf("%+v\n%+v\n", expectedValues, finalEvent.Results[0])
 	for matcherName, matchedValues := range expectedValues {
 		var matchesOne = false
 		for i := 0; i <= len(expectedValue); i++ {
@@ -262,24 +265,16 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 
-	matchedFileName := "test.txt"
 	fileContent := "test-data\r\n1.1.1.1\r\n"
+	matchedFileName := "test.txt"
+	input := "/tmp"
 
-	event := request.responseToDSLMap(&fileStatus{raw: fileContent, inputFilePath: "/tmp", matchedFileName: matchedFileName})
+	fileMatches := request.collectMatches(strings.NewReader(fileContent), input, matchedFileName, "")
-	require.Len(t, event, 11, "could not get correct number of items in dsl map")
+	event := request.buildEvent(input, matchedFileName, fileMatches)
-	require.Equal(t, fileContent, event["raw"], "could not get correct resp")
 
-	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
+	resultEvent := event.Results[0]
-	if request.CompiledOperators != nil {
-		result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug)
-		if ok && result != nil {
-			finalEvent.OperatorsResult = result
-			finalEvent.Results = request.MakeResultEvent(finalEvent)
-		}
-	}
-	resultEvent := finalEvent.Results[0]
 	require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
 	require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value")
 
-	return finalEvent
+	return event
 }

v2/pkg/protocols/file/request.go

@@ -65,153 +65,12 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 			if stat.Size() >= request.maxSize {
 				gologger.Verbose().Msgf("Limiting %s processed data to %s bytes: exceeded max size\n", filePath, units.HumanSize(float64(request.maxSize)))
 			}
-			totalBytes := units.BytesSize(float64(stat.Size()))
+
 			fileReader := io.LimitReader(file, request.maxSize)
-			var bytesCount, linesCount, wordsCount int
+			fileMatches := request.collectMatches(fileReader, input, filePath, units.BytesSize(float64(stat.Size())))
-			isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
-			scanner := bufio.NewScanner(fileReader)
-			buffer := []byte{}
-			scanner.Buffer(buffer, int(chunkSize))
-
-			var fileMatches []FileMatch
-			exprLines := make(map[string][]int)
-			exprBytes := make(map[string][]int)
-			for scanner.Scan() {
-				lineContent := scanner.Text()
-				n := len(lineContent)
-
-				// update counters
-				currentBytes := bytesCount + n
-				processedBytes := units.BytesSize(float64(currentBytes))
-
-				gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytes)
-				dslMap := request.responseToDSLMap(&fileStatus{
-					raw:             lineContent,
-					inputFilePath:   input,
-					matchedFileName: filePath,
-					lines:           linesCount,
-					words:           wordsCount,
-					bytes:           bytesCount,
-				})
-
-				if parts, ok := request.CompiledOperators.Execute(dslMap, request.Match, request.Extract, isResponseDebug); parts != nil && ok {
-					if parts.Extracts != nil {
-						for expr, extracts := range parts.Extracts {
-							for _, extract := range extracts {
-								fileMatches = append(fileMatches, FileMatch{
-									Data:      extract,
-									Extract:   true,
-									Line:      linesCount + 1,
-									ByteIndex: bytesCount,
-									Expr:      expr,
-									Raw:       lineContent,
-								})
-							}
-						}
-					}
-					if parts.Matches != nil {
-						for expr, matches := range parts.Matches {
-							for _, match := range matches {
-								fileMatches = append(fileMatches, FileMatch{
-									Data:      match,
-									Match:     true,
-									Line:      linesCount + 1,
-									ByteIndex: bytesCount,
-									Expr:      expr,
-									Raw:       lineContent,
-								})
-							}
-						}
-					}
-				}
-
-				currentLinesCount := 1 + strings.Count(lineContent, "\n")
-				linesCount += currentLinesCount
-				wordsCount += strings.Count(lineContent, " ")
-				bytesCount = currentBytes
-
-			}
 
 			// build event structure to interface with internal logic
-			internalEvent := request.responseToDSLMap(&fileStatus{
+			event := request.buildEvent(input, filePath, fileMatches)
-				inputFilePath:   input,
-				matchedFileName: filePath,
-			})
-			operatorResult := &operators.Result{}
-			for _, fileMatch := range fileMatches {
-				operatorResult.Matched = operatorResult.Matched || fileMatch.Match
-				operatorResult.Extracted = operatorResult.Extracted || fileMatch.Extract
-				switch {
-				case fileMatch.Extract:
-					if operatorResult.Extracts == nil {
-						operatorResult.Extracts = make(map[string][]string)
-					}
-					if _, ok := operatorResult.Extracts[fileMatch.Expr]; !ok {
-						operatorResult.Extracts[fileMatch.Expr] = []string{fileMatch.Data}
-					} else {
-						operatorResult.Extracts[fileMatch.Expr] = append(operatorResult.Extracts[fileMatch.Expr], fileMatch.Data)
-					}
-					operatorResult.OutputExtracts = append(operatorResult.OutputExtracts, fileMatch.Data)
-					if operatorResult.OutputUnique == nil {
-						operatorResult.OutputUnique = make(map[string]struct{})
-					}
-					operatorResult.OutputUnique[fileMatch.Data] = struct{}{}
-				case fileMatch.Match:
-					if operatorResult.Matches == nil {
-						operatorResult.Matches = make(map[string][]string)
-					}
-					if _, ok := operatorResult.Matches[fileMatch.Expr]; !ok {
-						operatorResult.Matches[fileMatch.Expr] = []string{fileMatch.Data}
-					} else {
-						operatorResult.Matches[fileMatch.Expr] = append(operatorResult.Matches[fileMatch.Expr], fileMatch.Data)
-					}
-				}
-				exprLines[fileMatch.Expr] = append(exprLines[fileMatch.Expr], fileMatch.Line)
-				exprBytes[fileMatch.Expr] = append(exprBytes[fileMatch.Expr], fileMatch.ByteIndex)
-			}
-
-			// build results
-			var results []*output.ResultEvent
-			for expr, items := range operatorResult.Matches {
-				results = append(results, &output.ResultEvent{
-					MatcherStatus:    true,
-					TemplateID:       types.ToString(internalEvent["template-id"]),
-					TemplatePath:     types.ToString(internalEvent["template-path"]),
-					Info:             internalEvent["template-info"].(model.Info),
-					Type:             types.ToString(internalEvent["type"]),
-					Path:             types.ToString(internalEvent["path"]),
-					Matched:          types.ToString(internalEvent["path"]),
-					Host:             types.ToString(internalEvent["host"]),
-					ExtractedResults: items,
-					// Response:         types.ToString(wrapped.InternalEvent["raw"]),
-					Timestamp:   time.Now(),
-					Lines:       exprLines[expr],
-					MatcherName: expr,
-				})
-			}
-			for expr, items := range operatorResult.Extracts {
-				results = append(results, &output.ResultEvent{
-					MatcherStatus:    true,
-					TemplateID:       types.ToString(internalEvent["template-id"]),
-					TemplatePath:     types.ToString(internalEvent["template-path"]),
-					Info:             internalEvent["template-info"].(model.Info),
-					Type:             types.ToString(internalEvent["type"]),
-					Path:             types.ToString(internalEvent["path"]),
-					Matched:          types.ToString(internalEvent["matched"]),
-					Host:             types.ToString(internalEvent["host"]),
-					ExtractedResults: items,
-					Lines:            exprLines[expr],
-					ExtractorName:    expr,
-					// FileToIndexPosition: exprBytes,
-					Timestamp: time.Now(),
-				})
-			}
-
-			event := &output.InternalWrappedEvent{
-				InternalEvent:   internalEvent,
-				Results:         results,
-				OperatorsResult: operatorResult,
-			}
 			dumpResponse(event, request.options, fileMatches, filePath)
 			callback(event)
 			request.options.Progress.IncrementRequests()
@@ -226,6 +85,144 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
 	return nil
 }
 
+func (request *Request) collectMatches(reader io.Reader, input, filePath, totalBytes string) []FileMatch {
+	var bytesCount, linesCount, wordsCount int
+	isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse
+	scanner := bufio.NewScanner(reader)
+	buffer := []byte{}
+	scanner.Buffer(buffer, int(chunkSize))
+
+	var fileMatches []FileMatch
+	for scanner.Scan() {
+		lineContent := scanner.Text()
+		n := len(lineContent)
+
+		// update counters
+		currentBytes := bytesCount + n
+		processedBytes := units.BytesSize(float64(currentBytes))
+
+		gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytes)
+		dslMap := request.responseToDSLMap(lineContent, input, filePath)
+		if parts, ok := request.CompiledOperators.Execute(dslMap, request.Match, request.Extract, isResponseDebug); parts != nil && ok {
+			if parts.Extracts != nil {
+				for expr, extracts := range parts.Extracts {
+					for _, extract := range extracts {
+						fileMatches = append(fileMatches, FileMatch{
+							Data:      extract,
+							Extract:   true,
+							Line:      linesCount + 1,
+							ByteIndex: bytesCount,
+							Expr:      expr,
+							Raw:       lineContent,
+						})
+					}
+				}
+			}
+			if parts.Matches != nil {
+				for expr, matches := range parts.Matches {
+					for _, match := range matches {
+						fileMatches = append(fileMatches, FileMatch{
+							Data:      match,
+							Match:     true,
+							Line:      linesCount + 1,
+							ByteIndex: bytesCount,
+							Expr:      expr,
+							Raw:       lineContent,
+						})
+					}
+				}
+			}
+		}
+
+		currentLinesCount := 1 + strings.Count(lineContent, "\n")
+		linesCount += currentLinesCount
+		wordsCount += strings.Count(lineContent, " ")
+		bytesCount = currentBytes
+	}
+	return fileMatches
+}
+
+func (request *Request) buildEvent(input, filePath string, fileMatches []FileMatch) *output.InternalWrappedEvent {
+	exprLines := make(map[string][]int)
+	exprBytes := make(map[string][]int)
+	internalEvent := request.responseToDSLMap("", input, filePath)
+	operatorResult := &operators.Result{}
+	for _, fileMatch := range fileMatches {
+		operatorResult.Matched = operatorResult.Matched || fileMatch.Match
+		operatorResult.Extracted = operatorResult.Extracted || fileMatch.Extract
+		switch {
+		case fileMatch.Extract:
+			if operatorResult.Extracts == nil {
+				operatorResult.Extracts = make(map[string][]string)
+			}
+			if _, ok := operatorResult.Extracts[fileMatch.Expr]; !ok {
+				operatorResult.Extracts[fileMatch.Expr] = []string{fileMatch.Data}
+			} else {
+				operatorResult.Extracts[fileMatch.Expr] = append(operatorResult.Extracts[fileMatch.Expr], fileMatch.Data)
+			}
+			operatorResult.OutputExtracts = append(operatorResult.OutputExtracts, fileMatch.Data)
+			if operatorResult.OutputUnique == nil {
+				operatorResult.OutputUnique = make(map[string]struct{})
+			}
+			operatorResult.OutputUnique[fileMatch.Data] = struct{}{}
+		case fileMatch.Match:
+			if operatorResult.Matches == nil {
+				operatorResult.Matches = make(map[string][]string)
+			}
+			if _, ok := operatorResult.Matches[fileMatch.Expr]; !ok {
+				operatorResult.Matches[fileMatch.Expr] = []string{fileMatch.Data}
+			} else {
+				operatorResult.Matches[fileMatch.Expr] = append(operatorResult.Matches[fileMatch.Expr], fileMatch.Data)
+			}
+		}
+		exprLines[fileMatch.Expr] = append(exprLines[fileMatch.Expr], fileMatch.Line)
+		exprBytes[fileMatch.Expr] = append(exprBytes[fileMatch.Expr], fileMatch.ByteIndex)
+	}
+
+	// build results
+	var results []*output.ResultEvent
+	for expr, items := range operatorResult.Matches {
+		results = append(results, &output.ResultEvent{
+			MatcherStatus:    true,
+			TemplateID:       types.ToString(internalEvent["template-id"]),
+			TemplatePath:     types.ToString(internalEvent["template-path"]),
+			Info:             internalEvent["template-info"].(model.Info),
+			Type:             types.ToString(internalEvent["type"]),
+			Path:             types.ToString(internalEvent["path"]),
+			Matched:          types.ToString(internalEvent["matched"]),
+			Host:             types.ToString(internalEvent["host"]),
+			ExtractedResults: items,
+			// Response:         types.ToString(wrapped.InternalEvent["raw"]),
+			Timestamp:   time.Now(),
+			Lines:       exprLines[expr],
+			MatcherName: expr,
+		})
+	}
+	for expr, items := range operatorResult.Extracts {
+		results = append(results, &output.ResultEvent{
+			MatcherStatus:    true,
+			TemplateID:       types.ToString(internalEvent["template-id"]),
+			TemplatePath:     types.ToString(internalEvent["template-path"]),
+			Info:             internalEvent["template-info"].(model.Info),
+			Type:             types.ToString(internalEvent["type"]),
+			Path:             types.ToString(internalEvent["path"]),
+			Matched:          types.ToString(internalEvent["matched"]),
+			Host:             types.ToString(internalEvent["host"]),
+			ExtractedResults: items,
+			Lines:            exprLines[expr],
+			ExtractorName:    expr,
+			// FileToIndexPosition: exprBytes,
+			Timestamp: time.Now(),
+		})
+	}
+
+	return &output.InternalWrappedEvent{
+		InternalEvent:   internalEvent,
+		Results:         results,
+		OperatorsResult: operatorResult,
+	}
+}
+
 func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, filematches []FileMatch, filePath string) {
 	cliOptions := requestOptions.Options
 	if cliOptions.Debug || cliOptions.DebugResponse {