feat: add osv-scanner for extracting lockfiles

2026-08-03 07:24:09 +02:00 · 2025-06-10 22:22:09 +05:30
parent 7ab669e623
commit a5ae1e50c0
6 changed files with 300 additions and 357 deletions
@@ -0,0 +1,101 @@
+package extractor
+
+import (
+	"fmt"
+
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+	"github.com/google/osv-scanner/pkg/lockfile"
+)
+
+// EcosystemExtractor defines the interface for ecosystem-specific extractors
+type EcosystemExtractor interface {
+	GetSupportedFiles() []string
+	GetEcosystem() packagev1.Ecosystem
+	Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error)
+}
+
+// NpmExtractor handles NPM ecosystem lockfiles
+type NpmExtractor struct{}
+
+var NpmExtractors []string = []string{"package-lock.json", "pnpm-lock.yaml"}
+
+func (n *NpmExtractor) GetSupportedFiles() []string {
+	return NpmExtractors
+}
+
+func (n *NpmExtractor) GetEcosystem() packagev1.Ecosystem {
+	return packagev1.Ecosystem_ECOSYSTEM_NPM
+}
+
+func (n *NpmExtractor) Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error) {
+	switch filename {
+	case "package-lock.json":
+		extractor := lockfile.NpmLockExtractor{}
+		return extractor.Extract(depFile)
+	case "pnpm-lock.yaml":
+		extractor := lockfile.PnpmLockExtractor{}
+		return extractor.Extract(depFile)
+	default:
+		return nil, fmt.Errorf("unsupported NPM lockfile: %s", filename)
+	}
+}
+
+// PyPiExtractor handles PyPI ecosystem lockfiles
+type PyPiExtractor struct{}
+
+var PyPiExtractors []string = []string{"requirements.txt", "Pipfile.lock"}
+
+func (p *PyPiExtractor) GetSupportedFiles() []string {
+	return PyPiExtractors
+}
+
+func (p *PyPiExtractor) GetEcosystem() packagev1.Ecosystem {
+	return packagev1.Ecosystem_ECOSYSTEM_PYPI
+}
+
+func (p *PyPiExtractor) Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error) {
+	switch filename {
+	case "requirements.txt":
+		extractor := lockfile.RequirementsTxtExtractor{}
+		return extractor.Extract(depFile)
+	case "Pipfile.lock":
+		extractor := lockfile.PipenvLockExtractor{}
+		return extractor.Extract(depFile)
+	default:
+		return nil, fmt.Errorf("unsupported PyPI lockfile: %s", filename)
+	}
+}
+
+// ExtractorManager manages all ecosystem extractors
+type ExtractorManager struct {
+	extractors map[packagev1.Ecosystem]EcosystemExtractor
+}
+
+func NewExtractorManager() *ExtractorManager {
+	return &ExtractorManager{
+		extractors: map[packagev1.Ecosystem]EcosystemExtractor{
+			packagev1.Ecosystem_ECOSYSTEM_NPM:  &NpmExtractor{},
+			packagev1.Ecosystem_ECOSYSTEM_PYPI: &PyPiExtractor{},
+		},
+	}
+}
+
+func (em *ExtractorManager) GetExtractorForEcosystem(ecosystem packagev1.Ecosystem) (EcosystemExtractor, bool) {
+	extractor, exists := em.extractors[ecosystem]
+	return extractor, exists
+}
+
+func (em *ExtractorManager) GetSupportedFilesForEcosystem(ecosystem packagev1.Ecosystem) []string {
+	if extractor, exists := em.extractors[ecosystem]; exists {
+		return extractor.GetSupportedFiles()
+	}
+	return nil
+}
+
+func (em *ExtractorManager) GetAllSupportedFiles() []string {
+	var allFiles []string
+	for _, extractor := range em.extractors {
+		allFiles = append(allFiles, extractor.GetSupportedFiles()...)
+	}
+	return allFiles
+}
@@ -0,0 +1,159 @@
+package extractor
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+	"github.com/google/osv-scanner/pkg/lockfile"
+)
+
+type ExtractorConfig struct {
+	// List of extractors used by an ecosystem
+	ExtractorsName     []string
+	ExtractorEcosystem packagev1.Ecosystem
+	ScanDir            string
+}
+
+type extractor struct {
+	Config           ExtractorConfig
+	extractorManager *ExtractorManager
+}
+
+func NewDefaultExtractorConfig() *ExtractorConfig {
+	return &ExtractorConfig{
+		ScanDir: ".",
+	}
+}
+
+func New(config ExtractorConfig) *extractor {
+	return &extractor{
+		Config:           config,
+		extractorManager: NewExtractorManager(),
+	}
+}
+
+func (e *extractor) ExtractManifest() ([]*packagev1.PackageVersion, error) {
+	packagesToAnalyze := []*packagev1.PackageVersion{}
+
+	// Get the list of lockfiles to check based on ecosystem
+	filesToCheck := e.getFilesToCheck()
+
+	for _, filename := range filesToCheck {
+		filePath := filepath.Join(e.Config.ScanDir, filename)
+
+		// Check if the file exists
+		if _, err := os.Stat(filePath); os.IsNotExist(err) {
+			continue
+		}
+
+		// Extract packages from this lockfile
+		packages, err := e.extractFromLockfile(filePath)
+		if err != nil {
+			fmt.Printf("Warning: failed to extract from %s: %v\n", filePath, err)
+			continue
+		}
+
+		packagesToAnalyze = append(packagesToAnalyze, packages...)
+	}
+
+	return packagesToAnalyze, nil
+}
+
+func (e *extractor) getFilesToCheck() []string {
+	// If specific extractors are configured, use those
+	if len(e.Config.ExtractorsName) > 0 {
+		return e.Config.ExtractorsName
+	}
+
+	// If ecosystem is specified, return files for that ecosystem
+	if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
+		return e.extractorManager.GetSupportedFilesForEcosystem(e.Config.ExtractorEcosystem)
+	}
+
+	// If no ecosystem specified, check all known extractors
+	return e.extractorManager.GetAllSupportedFiles()
+}
+
+func (e *extractor) extractFromLockfile(path string) ([]*packagev1.PackageVersion, error) {
+	filename := filepath.Base(path)
+
+	// Find the appropriate extractor for this file
+	ecosystemExtractor, ecosystem, err := e.getExtractorForFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	// Open the file as a DepFile for extraction
+	depFile, err := lockfile.OpenLocalDepFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %s: %w", path, err)
+	}
+	defer depFile.Close()
+
+	// Extract dependencies using the appropriate extractor
+	deps, err := ecosystemExtractor.Extract(depFile, filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract from %s: %w", path, err)
+	}
+
+	packages := e.convertToPackageVersions(deps, ecosystem)
+	return packages, nil
+}
+
+func (e *extractor) getExtractorForFile(filename string) (EcosystemExtractor, packagev1.Ecosystem, error) {
+	// If ecosystem is configured, use that directly
+	if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
+		extractor, exists := e.extractorManager.GetExtractorForEcosystem(e.Config.ExtractorEcosystem)
+		if !exists {
+			return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("no extractor found for ecosystem: %v", e.Config.ExtractorEcosystem)
+		}
+
+		// Verify the file is supported by this ecosystem
+		supportedFiles := extractor.GetSupportedFiles()
+		for _, supportedFile := range supportedFiles {
+			if filename == supportedFile || strings.Contains(filename, supportedFile) {
+				return extractor, e.Config.ExtractorEcosystem, nil
+			}
+		}
+		return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("file %s not supported by ecosystem %v", filename, e.Config.ExtractorEcosystem)
+	}
+
+	// Try to find extractor by filename
+	for ecosystem, extractor := range e.extractorManager.extractors {
+		supportedFiles := extractor.GetSupportedFiles()
+		for _, supportedFile := range supportedFiles {
+			if filename == supportedFile || strings.Contains(filename, supportedFile) {
+				return extractor, ecosystem, nil
+			}
+		}
+	}
+
+	return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("unsupported lockfile type: %s", filename)
+}
+
+func (e *extractor) convertToPackageVersions(deps []lockfile.PackageDetails, ecosystem packagev1.Ecosystem) []*packagev1.PackageVersion {
+	var packages []*packagev1.PackageVersion
+
+	for _, dep := range deps {
+		// Use the configured ecosystem if provided, otherwise use the detected one
+		targetEcosystem := ecosystem
+		if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
+			targetEcosystem = e.Config.ExtractorEcosystem
+		}
+
+		packageVer := &packagev1.PackageVersion{
+			Package: &packagev1.Package{
+				Name:      dep.Name,
+				Ecosystem: targetEcosystem,
+			},
+			Version: dep.Version,
+		}
+
+		packages = append(packages, packageVer)
+	}
+
+	return packages
+}