Added preprocessors + bug fixes

v2/pkg/protocols/network/operators.go

@@ -62,7 +62,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
 }
 
 // responseToDSLMap converts a DNS response to a map for use in DSL matching
-func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent {
+func (r *Request) responseToDSLMap(req, resp, raw string, host, matched string) output.InternalEvent {
 	data := make(output.InternalEvent, 6)
 
 	// Some data regarding the request metadata
@@ -70,6 +70,7 @@ func (r *Request) responseToDSLMap(req, resp string, host, matched string) outpu
 	data["matched"] = matched
 	data["request"] = req
 	data["data"] = resp
+	data["raw"] = raw
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
 	return data

v2/pkg/protocols/network/operators_test.go

@@ -31,7 +31,7 @@ func TestResponseToDSLMap(t *testing.T) {
 
 	req := "test-data\r\n"
 	resp := "resp-data\r\n"
-	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test")
 	require.Len(t, event, 6, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["data"], "could not get correct resp")
 }
@@ -56,7 +56,7 @@ func TestNetworkOperatorMatch(t *testing.T) {
 
 	req := "test-data\r\n"
 	resp := "resp-data\r\nSTAT \r\n"
-	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test")
 
 	t.Run("valid", func(t *testing.T) {
 		matcher := &matchers.Matcher{
@@ -119,7 +119,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
 
 	req := "test-data\r\n"
 	resp := "resp-data\r\nSTAT \r\n1.1.1.1\r\n"
-	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test")
 
 	t.Run("extract", func(t *testing.T) {
 		extractor := &extractors.Extractor{
@@ -182,7 +182,7 @@ func TestNetworkMakeResult(t *testing.T) {
 
 	req := "test-data\r\n"
 	resp := "resp-data\rSTAT \r\n1.1.1.1\n"
-	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
+	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test")
 	finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
 	event["ip"] = "192.168.1.1"
 	if request.CompiledOperators != nil {

v2/pkg/protocols/network/request.go

@@ -99,7 +99,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, previous
 			return errors.Wrap(err, "could not write request to server")
 		}
 
-		if r.ReadSize != 0 {
+		if input.Read > 0 {
 			buffer := make([]byte, r.ReadSize)
 			n, _ := conn.Read(buffer)
 			responseBuilder.Write(buffer[:n])
@@ -124,15 +124,15 @@ func (r *Request) executeAddress(actualAddress, address, input string, previous
 	if r.ReadSize != 0 {
 		bufferSize = r.ReadSize
 	}
-	buffer = make([]byte, bufferSize)
+	final := make([]byte, bufferSize)
-	n, _ = conn.Read(buffer)
+	n, _ = conn.Read(final)
-	responseBuilder.Write(buffer[:n])
+	responseBuilder.Write(final[:n])
 
 	if r.options.Options.Debug || r.options.Options.DebugResponse {
 		gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
 		gologger.Print().Msgf("%s", responseBuilder.String())
 	}
-	outputEvent := r.responseToDSLMap(reqBuilder.String(), responseBuilder.String(), input, actualAddress)
+	outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress)
 	outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
 	for k, v := range previous {
 		outputEvent[k] = v

v2/pkg/templates/preprocessors.go

@@ -0,0 +1,32 @@
+package templates
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"github.com/segmentio/ksuid"
+)
+
+var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`)
+
+// expandPreprocessors expands the pre-processors if any for a template data.
+func (t *Template) expandPreprocessors(data []byte) []byte {
+	foundMap := make(map[string]struct{})
+
+	for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) {
+		if len(expression) != 2 {
+			continue
+		}
+		value := expression[1]
+
+		if _, ok := foundMap[value]; ok {
+			continue
+		}
+		foundMap[value] = struct{}{}
+		if strings.EqualFold(value, "randstr") || strings.HasPrefix(value, "randstr_") {
+			data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(ksuid.New().String()))
+		}
+	}
+	return data
+}