fix: segfault in template caching logic (#6421)

* fix: segfault in template caching logic

when templates had no executable requests after
option updates.

the cached templates could end up with 0 requests
and no flow execution path, resulting in a nil
engine pointer that was later derefer w/o
validation.

bug seq:
caching template (w/ valid requests) -> get cached
template -> `*ExecutorOptions.Options` copied and
modified (inconsistent) -> requests updated (with
new options -- some may be invalid, and without
recompile) -> template returned w/o validation ->
`compileProtocolRequests` -> `NewTemplateExecuter`
receive empty requests + empty flow = nil engine
-> `*TemplateExecuter.{Compile,Execute}` invoked
on nil engine = panic.

RCA:
1. `*ExecutorOptions.ApplyNewEngineOptions`
   overwriting many fields.
2. copy op pointless; create a copy of options and
   then immediately replace it with original
   pointer.
3. missing executable requests validation after
   cached templates is reconstructed with updated
   options.

Thus, this affected `--automatic-scan` mode where
tech detection templates often have conditional
requests that may be filtered based on runtime
options.

Fixes #6417

Signed-off-by: Dwi Siswanto <git@dw1.io>

* fix(templates): recompile workflow with `tplCopy.Options`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* fix(templates): strengthen cache hit guard

Signed-off-by: Dwi Siswanto <git@dw1.io>

* fix(protocols): skips template-specific fields

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>

pkg/protocols/protocols.go

@@ -447,21 +447,8 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 		return
 	}
 
-	// The types.Options include the ExecutionID among other things
 	e.Options = n.Options.Copy()
-
-	// Keep the template-specific fields, but replace the rest
-	/*
-		e.TemplateID = n.TemplateID
-		e.TemplatePath = n.TemplatePath
-		e.TemplateInfo = n.TemplateInfo
-		e.TemplateVerifier = n.TemplateVerifier
-		e.RawTemplate = n.RawTemplate
-		e.Variables = n.Variables
-		e.Constants = n.Constants
-	*/
 	e.Output = n.Output
-	e.Options = n.Options
 	e.IssuesClient = n.IssuesClient
 	e.Progress = n.Progress
 	e.RateLimiter = n.RateLimiter
@@ -470,8 +457,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	e.Browser = n.Browser
 	e.Interactsh = n.Interactsh
 	e.HostErrorsCache = n.HostErrorsCache
-	e.StopAtFirstMatch = n.StopAtFirstMatch
-	e.ExcludeMatchers = n.ExcludeMatchers
 	e.InputHelper = n.InputHelper
 	e.FuzzParamsFrequency = n.FuzzParamsFrequency
 	e.FuzzStatsDB = n.FuzzStatsDB
@@ -479,10 +464,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	e.Colorizer = n.Colorizer
 	e.WorkflowLoader = n.WorkflowLoader
 	e.ResumeCfg = n.ResumeCfg
-	e.ProtocolType = n.ProtocolType
-	e.Flow = n.Flow
-	e.IsMultiProtocol = n.IsMultiProtocol
-	e.templateCtxStore = n.templateCtxStore
 	e.JsCompiler = n.JsCompiler
 	e.AuthProvider = n.AuthProvider
 	e.TemporaryDirectory = n.TemporaryDirectory

pkg/templates/compile.go

@@ -64,6 +64,13 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut
 			newBase.TemplateInfo = tplCopy.Options.TemplateInfo
 			newBase.TemplateVerifier = tplCopy.Options.TemplateVerifier
 			newBase.RawTemplate = tplCopy.Options.RawTemplate
+
+			if tplCopy.Options.Variables.Len() > 0 {
+				newBase.Variables = tplCopy.Options.Variables
+			}
+			if len(tplCopy.Options.Constants) > 0 {
+				newBase.Constants = tplCopy.Options.Constants
+			}
 			tplCopy.Options = newBase
 
 			tplCopy.Options.ApplyNewEngineOptions(options)
@@ -156,12 +163,16 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut
 			// Compile the workflow request
 			if len(template.Workflows) > 0 {
 				compiled := &template.Workflow
-				compileWorkflow(filePath, preprocessor, options, compiled, options.WorkflowLoader)
+				compileWorkflow(filePath, preprocessor, tplCopy.Options, compiled, tplCopy.Options.WorkflowLoader)
 				template.CompiledWorkflow = compiled
-				template.CompiledWorkflow.Options = options
+				template.CompiledWorkflow.Options = tplCopy.Options
 			}
-			// options.Logger.Error().Msgf("returning cached template %s after recompiling %d requests", tplCopy.Options.TemplateID, tplCopy.Requests())
-			return template, nil
+
+			if isCachedTemplateValid(template) {
+				// options.Logger.Error().Msgf("returning cached template %s after recompiling %d requests", tplCopy.Options.TemplateID, tplCopy.Requests())
+				return template, nil
+			}
+			// else: fallthrough to re-parse template from scratch
 		}
 	}
 
@@ -579,6 +590,50 @@ func parseTemplate(data []byte, srcOptions *protocols.ExecutorOptions) (*Templat
 	return template, nil
 }
 
+// isCachedTemplateValid validates that a cached template is still usable after
+// option updates
+func isCachedTemplateValid(template *Template) bool {
+	// no requests or workflows
+	if template.Requests() == 0 && len(template.Workflows) == 0 {
+		return false
+	}
+
+	// options not initialized
+	if template.Options == nil {
+		return false
+	}
+
+	// executer not available for non-workflow template
+	if len(template.Workflows) == 0 && template.Executer == nil {
+		return false
+	}
+
+	// compiled workflow not available
+	if len(template.Workflows) > 0 && template.CompiledWorkflow == nil {
+		return false
+	}
+
+	// template ID mismatch
+	if template.Options.TemplateID != template.ID {
+		return false
+	}
+
+	// executer exists but no requests or flow available
+	if template.Executer != nil {
+		// NOTE(dwisiswant0): This is a basic sanity check since we can't access
+		// private fields, but we can check requests tho
+		if template.Requests() == 0 && template.Options.Flow == "" {
+			return false
+		}
+	}
+
+	if template.Options.Options == nil {
+		return false
+	}
+
+	return true
+}
+
 var (
 	jsCompiler     *compiler.Compiler
 	jsCompilerOnce = sync.OnceFunc(func() {