Support for scanning manifest files (#36)

* feat: add extractor logic for manifest files * feat: add manifest-based package installation detection * feat: add osv-scanner for extracting lockfiles * refactor: add comment for cmd parse * refactor: add Ecosystem() method to PackageManager interface * refactor: implement package-manager-specific extractors & replace osv-scanner with scalibr * Update extractor/extractor.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Sahil Bansal <bansalsahil315@gmail.com> --------- Signed-off-by: Sahil Bansal <bansalsahil315@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-08-03 07:24:09 +02:00 · 2025-06-12 18:58:58 +05:30
parent 302e17fe48
commit 5e9f2af972
13 changed files with 690 additions and 16 deletions
@@ -0,0 +1,50 @@
+package extractor
+
+import (
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+)
+
+// PackageManagerExtractor defines the interface for package-manager-specific extractors
+type PackageManagerExtractor interface {
+	// Returns the list of supported lockfiles by the package manager
+	GetSupportedFiles() []string
+
+	// Returns the package manager ecosysetm
+	GetEcosystem() packagev1.Ecosystem
+
+	// Returns the package manager name
+	GetPackageManager() PackageManagerName
+
+	// Extracts the packages from the lockfile
+	Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error)
+}
+
+type PackageManagerName string
+
+const (
+	Npm  PackageManagerName = "npm"
+	Pnpm PackageManagerName = "pnpm"
+	Pip  PackageManagerName = "pip"
+)
+
+type ExtractorManager struct {
+	extractors map[PackageManagerName]PackageManagerExtractor
+}
+
+func NewExtractorManager() *ExtractorManager {
+	return &ExtractorManager{
+		extractors: map[PackageManagerName]PackageManagerExtractor{
+			Npm:  &NpmExtractor{},
+			Pnpm: &PnpmExtractor{},
+			Pip:  &PipExtractor{},
+		},
+	}
+}
+
+func (e *ExtractorManager) GetExtractorForPackageManager(pmn PackageManagerName) PackageManagerExtractor {
+	return e.extractors[pmn]
+}
+
+func (e *ExtractorManager) GetSupportedFilesForPackageManager(pmn PackageManagerName) []string {
+	return e.extractors[pmn].GetSupportedFiles()
+}
@@ -0,0 +1,78 @@
+package extractor
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+	"github.com/safedep/dry/log"
+)
+
+type ExtractorConfig struct {
+	ExtractorPackageManager PackageManagerName
+	ScanDir                 string
+}
+
+type extractor struct {
+	Config           ExtractorConfig
+	extractorManager ExtractorManager
+}
+
+func NewDefaultExtractorConfig() *ExtractorConfig {
+	return &ExtractorConfig{
+		ScanDir: ".",
+	}
+}
+
+func New(config ExtractorConfig) *extractor {
+	return &extractor{
+		Config:           config,
+		extractorManager: *NewExtractorManager(),
+	}
+}
+
+func (e *extractor) ExtractManifest() ([]*packagev1.PackageVersion, error) {
+	packagesToAnalyze := []*packagev1.PackageVersion{}
+
+	// Get the list of lockfiles to check based on ecosystem
+	filesToCheck := e.getFilesToCheck()
+
+	for _, filename := range filesToCheck {
+		filePath := filepath.Join(e.Config.ScanDir, filename)
+
+		// Check if the file exists
+		if _, err := os.Stat(filePath); os.IsNotExist(err) {
+			continue
+		}
+
+		extractor, err := e.getExtractorForFile()
+		if err != nil {
+			log.Warnf("failed to get extractor\n")
+			continue
+		}
+
+		// Extract packages from this lockfile
+		packages, err := extractor.Extract(filePath, e.Config.ScanDir)
+		if err != nil {
+			log.Warnf("failed to extract from %s: %v\n", filePath, err)
+			continue
+		}
+
+		packagesToAnalyze = append(packagesToAnalyze, packages...)
+	}
+
+	return packagesToAnalyze, nil
+}
+
+func (e *extractor) getFilesToCheck() []string {
+	return e.extractorManager.GetSupportedFilesForPackageManager(e.Config.ExtractorPackageManager)
+}
+
+func (e *extractor) getExtractorForFile() (PackageManagerExtractor, error) {
+	extractor := e.extractorManager.GetExtractorForPackageManager(e.Config.ExtractorPackageManager)
+	if extractor == nil {
+		return nil, fmt.Errorf("no extractor found for the specified package manager: %s", e.Config.ExtractorPackageManager)
+	}
+	return extractor, nil
+}
@@ -0,0 +1,125 @@
+package extractor
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock"
+	"github.com/google/osv-scalibr/fs"
+)
+
+// NpmExtractor handles package-lock.json files
+type NpmExtractor struct{}
+
+func (n *NpmExtractor) GetSupportedFiles() []string {
+	return []string{"package-lock.json"}
+}
+
+func (n *NpmExtractor) GetEcosystem() packagev1.Ecosystem {
+	return packagev1.Ecosystem_ECOSYSTEM_NPM
+}
+
+func (n *NpmExtractor) GetPackageManager() PackageManagerName {
+	return Npm
+}
+
+func (n *NpmExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	return parseNpmPackageLockFile(lockfilePath, scanDir)
+}
+
+func parseNpmPackageLockFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	packagelockExtractor := packagelockjson.NewDefault()
+
+	file, err := os.Open(lockfilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open lockfile: %s", err)
+	}
+	defer file.Close()
+
+	inputConfig := &filesystem.ScanInput{
+		FS:     fs.DirFS(scanDir),
+		Path:   lockfilePath,
+		Reader: file,
+	}
+
+	inventory, err := packagelockExtractor.Extract(context.Background(), inputConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract packages: %s", err)
+	}
+
+	var packages []*packagev1.PackageVersion
+
+	for _, invPkg := range inventory.Packages {
+		pkg := &packagev1.PackageVersion{
+			Package: &packagev1.Package{
+				Name:      invPkg.Name,
+				Ecosystem: packagev1.Ecosystem_ECOSYSTEM_NPM,
+			},
+			Version: invPkg.Version,
+		}
+
+		packages = append(packages, pkg)
+	}
+
+	return packages, nil
+}
+
+// PnpmExtractor handles pnpm-lock.yaml files
+type PnpmExtractor struct{}
+
+func (p *PnpmExtractor) GetSupportedFiles() []string {
+	return []string{"pnpm-lock.yaml"}
+}
+
+func (p *PnpmExtractor) GetEcosystem() packagev1.Ecosystem {
+	return packagev1.Ecosystem_ECOSYSTEM_NPM
+}
+
+func (p *PnpmExtractor) GetPackageManager() PackageManagerName {
+	return Pnpm
+}
+
+func (p *PnpmExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	return parsePnpmLockFile(lockfilePath, scanDir)
+}
+
+func parsePnpmLockFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	pnpmLockExtractor := pnpmlock.New()
+
+	file, err := os.Open(lockfilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open lockfile: %s", err)
+	}
+	defer file.Close()
+
+	inputConfig := &filesystem.ScanInput{
+		FS:     fs.DirFS(scanDir),
+		Path:   lockfilePath,
+		Reader: file,
+	}
+
+	inventory, err := pnpmLockExtractor.Extract(context.Background(), inputConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract packages: %s", err)
+	}
+
+	var packages []*packagev1.PackageVersion
+
+	for _, invPkg := range inventory.Packages {
+		pkg := &packagev1.PackageVersion{
+			Package: &packagev1.Package{
+				Name:      invPkg.Name,
+				Ecosystem: packagev1.Ecosystem_ECOSYSTEM_NPM,
+			},
+			Version: invPkg.Version,
+		}
+
+		packages = append(packages, pkg)
+	}
+
+	return packages, nil
+}
@@ -0,0 +1,68 @@
+package extractor
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements"
+	"github.com/google/osv-scalibr/fs"
+)
+
+// PipExtractor handles requirements.txt files
+type PipExtractor struct{}
+
+func (p *PipExtractor) GetSupportedFiles() []string {
+	return []string{"requirements.txt"}
+}
+
+func (p *PipExtractor) GetEcosystem() packagev1.Ecosystem {
+	return packagev1.Ecosystem_ECOSYSTEM_PYPI
+}
+
+func (p *PipExtractor) GetPackageManager() PackageManagerName {
+	return Pip
+}
+
+func (n *PipExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	return parseRequirementsTxtFile(lockfilePath, scanDir)
+}
+
+func parseRequirementsTxtFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
+	requirementsExtractor := requirements.NewDefault()
+
+	file, err := os.Open(lockfilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open lockfile: %s", err)
+	}
+	defer file.Close()
+
+	inputConfig := &filesystem.ScanInput{
+		FS:     fs.DirFS(scanDir),
+		Path:   lockfilePath,
+		Reader: file,
+	}
+
+	inventory, err := requirementsExtractor.Extract(context.Background(), inputConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract packages: %s", err)
+	}
+
+	var packages []*packagev1.PackageVersion
+
+	for _, invPkg := range inventory.Packages {
+		pkg := &packagev1.PackageVersion{
+			Package: &packagev1.Package{
+				Name:      invPkg.Name,
+				Ecosystem: packagev1.Ecosystem_ECOSYSTEM_PYPI,
+			},
+			Version: invPkg.Version,
+		}
+
+		packages = append(packages, pkg)
+	}
+
+	return packages, nil
+}