package disk import ( "io/fs" "os" "path/filepath" "strings" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" stringsutil "github.com/projectdiscovery/utils/strings" updateutils "github.com/projectdiscovery/utils/update" urlutil "github.com/projectdiscovery/utils/url" ) var deprecatedPathsCounter int // GetTemplatesPath returns a list of absolute paths for the provided template list. func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} erred := make(map[string]error) for _, t := range definitions { if stringsutil.ContainsAny(t, knownConfigFiles...) { // TODO: this is a temporary fix to avoid treating these files as templates // this should be replaced with more appropriate and robust logic continue } if strings.Contains(t, urlutil.SchemeSeparator) && stringsutil.ContainsAny(t, config.GetSupportTemplateFileExtensions()...) { if _, ok := processed[t]; !ok { processed[t] = true allTemplates = append(allTemplates, t) } } else { paths, err := c.GetTemplatePath(t) if err != nil { erred[t] = err } for _, path := range paths { if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) } } } } // purge all false positives filteredTemplates := []string{} for _, v := range allTemplates { // TODO: this is a temporary fix to avoid treating these files as templates // this should be replaced with more appropriate and robust logic if !stringsutil.ContainsAny(v, knownConfigFiles...) { filteredTemplates = append(filteredTemplates, v) } } return filteredTemplates, erred } // GetTemplatePath parses the specified input template path and returns a compiled // list of finished absolute paths to the templates evaluating any glob patterns // or folders provided as in. func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { processed := make(map[string]struct{}) // Template input includes a wildcard if strings.Contains(target, "*") { matches, findErr := c.findGlobPathMatches(target, processed) if findErr != nil { return nil, errors.Wrap(findErr, "could not find glob matches") } if len(matches) == 0 { return nil, errors.Errorf("no templates found for path") } return matches, nil } // try to handle deprecated template paths absPath := target if c.templatesFS == nil { absPath = BackwardsCompatiblePaths(c.templatesDirectory, target) if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target { if config.DefaultConfig.LogAllEvents { config.DefaultConfig.Logger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath) } deprecatedPathsCounter++ } var err error absPath, err = c.convertPathToAbsolute(absPath) if err != nil { return nil, errors.Wrapf(err, "could not find template file") } } // Template input is either a file or a directory match, file, err := c.findFileMatches(absPath, processed) if err != nil { return nil, errors.Wrap(err, "could not find file") } if file { if match != "" { return []string{match}, nil } return nil, nil } // Recursively walk down the Templates directory and run all // the template file checks matches, err := c.findDirectoryMatches(absPath, processed) if err != nil { return nil, errors.Wrap(err, "could not find directory matches") } if len(matches) == 0 { return nil, errors.Errorf("no templates found in path %s", absPath) } return matches, nil } // convertPathToAbsolute resolves the paths provided to absolute paths // before doing any operations on them regardless of them being BLOB, folders, files, etc. func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) { if strings.Contains(t, "*") { file := filepath.Base(t) absPath, err := c.ResolvePath(filepath.Dir(t), "") if err != nil { return "", err } return filepath.Join(absPath, file), nil } return c.ResolvePath(t, "") } // findGlobPathMatches returns the matched files from a glob path func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { // to support globbing on old paths we use brute force to find matches with exit on first match // trim templateDir if any relPath := strings.TrimPrefix(absPath, c.templatesDirectory) // trim leading slash if any relPath = strings.TrimPrefix(relPath, string(os.PathSeparator)) OldPathsResolver := func(inputGlob string) []string { templateDir := c.templatesDirectory if c.templatesDirectory == "" { templateDir = "./" } if c.templatesFS == nil { matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob) if len(matches) != 0 { return matches } // condition to support network cve related globs matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob) return matches } else { sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http")) if err != nil { return nil } matches, _ := fs.Glob(sub, inputGlob) if len(matches) != 0 { return matches } // condition to support network cve related globs sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network")) if err != nil { return nil } matches, _ = fs.Glob(sub, inputGlob) return matches } } var matched []string var matches []string if c.templatesFS == nil { var err error matches, err = filepath.Glob(relPath) if len(matches) != 0 { matched = append(matched, matches...) } else { matched = append(matched, OldPathsResolver(relPath)...) } if err != nil && len(matched) == 0 { return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) } } else { var err error matches, err = fs.Glob(c.templatesFS, relPath) if len(matches) != 0 { matched = append(matched, matches...) } else { matched = append(matched, OldPathsResolver(relPath)...) } if err != nil && len(matched) == 0 { return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) } } results := make([]string, 0, len(matches)) for _, match := range matches { if _, ok := processed[match]; !ok { processed[match] = struct{}{} results = append(results, match) } } return results, nil } // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { if c.templatesFS != nil { absPath = strings.Trim(absPath, "/") } var info fs.File if c.templatesFS == nil { info, err = os.Open(absPath) } else { // If we were given no path, then it's not a file, it's the root, and we can quietly return. if absPath == "" { return "", false, nil } info, err = c.templatesFS.Open(absPath) } if err != nil { return "", false, err } stat, err := info.Stat() if err != nil { return "", false, err } if !stat.Mode().IsRegular() { return "", false, nil } if _, ok := processed[absPath]; !ok { processed[absPath] = struct{}{} return absPath, true, nil } return "", true, nil } // findDirectoryMatches finds matches for templates from a directory func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { var results []string var err error if c.templatesFS == nil { err = filepath.WalkDir( absPath, func(path string, d fs.DirEntry, err error) error { // continue on errors if err != nil { return nil } if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} } } return nil }, ) } else { // For the special case of the root directory, we need to pass "." to `fs.WalkDir`. if absPath == "" { absPath = "." } absPath = strings.TrimSuffix(absPath, "/") err = fs.WalkDir( c.templatesFS, absPath, func(path string, d fs.DirEntry, err error) error { // continue on errors if err != nil { return nil } if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { if _, ok := processed[path]; !ok { results = append(results, path) processed[path] = struct{}{} } } return nil }, ) } return results, err } // PrintDeprecatedPathsMsgIfApplicable prints a warning message if any deprecated paths are found // Unless mode is silent warning message is printed func PrintDeprecatedPathsMsgIfApplicable(isSilent bool) { if !updateutils.IsOutdated("v9.4.3", config.DefaultConfig.TemplateVersion) { return } if deprecatedPathsCounter > 0 && !isSilent { config.DefaultConfig.Logger.Print().Msgf("[%v] Found %v template[s] loaded with deprecated paths, update before v3 for continued support.\n", aurora.Yellow("WRN").String(), deprecatedPathsCounter) } }