mirror of
https://github.com/safedep/pmg.git
synced 2026-08-03 07:24:09 +02:00
refactor: implement package-manager-specific extractors & replace osv-scanner with scalibr
This commit is contained in:
Sahilb315
+26
-77
@@ -1,101 +1,50 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
|
||||
"github.com/google/osv-scanner/pkg/lockfile"
|
||||
)
|
||||
|
||||
// EcosystemExtractor defines the interface for ecosystem-specific extractors
|
||||
type EcosystemExtractor interface {
|
||||
// PackageManagerExtractor defines the interface for package-manager-specific extractors
|
||||
type PackageManagerExtractor interface {
|
||||
// Returns the list of supported lockfiles by the package manager
|
||||
GetSupportedFiles() []string
|
||||
|
||||
// Returns the package manager ecosysetm
|
||||
GetEcosystem() packagev1.Ecosystem
|
||||
Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error)
|
||||
|
||||
// Returns the package manager name
|
||||
GetPackageManager() PackageManagerName
|
||||
|
||||
// Extracts the packages from the lockfile
|
||||
Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error)
|
||||
}
|
||||
|
||||
// NpmExtractor handles NPM ecosystem lockfiles
|
||||
type NpmExtractor struct{}
|
||||
type PackageManagerName string
|
||||
|
||||
var NpmExtractors []string = []string{"package-lock.json", "pnpm-lock.yaml"}
|
||||
const (
|
||||
Npm PackageManagerName = "npm"
|
||||
Pnpm PackageManagerName = "pnpm"
|
||||
Pip PackageManagerName = "pip"
|
||||
)
|
||||
|
||||
func (n *NpmExtractor) GetSupportedFiles() []string {
|
||||
return NpmExtractors
|
||||
}
|
||||
|
||||
func (n *NpmExtractor) GetEcosystem() packagev1.Ecosystem {
|
||||
return packagev1.Ecosystem_ECOSYSTEM_NPM
|
||||
}
|
||||
|
||||
func (n *NpmExtractor) Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error) {
|
||||
switch filename {
|
||||
case "package-lock.json":
|
||||
extractor := lockfile.NpmLockExtractor{}
|
||||
return extractor.Extract(depFile)
|
||||
case "pnpm-lock.yaml":
|
||||
extractor := lockfile.PnpmLockExtractor{}
|
||||
return extractor.Extract(depFile)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported NPM lockfile: %s", filename)
|
||||
}
|
||||
}
|
||||
|
||||
// PyPiExtractor handles PyPI ecosystem lockfiles
|
||||
type PyPiExtractor struct{}
|
||||
|
||||
var PyPiExtractors []string = []string{"requirements.txt", "Pipfile.lock"}
|
||||
|
||||
func (p *PyPiExtractor) GetSupportedFiles() []string {
|
||||
return PyPiExtractors
|
||||
}
|
||||
|
||||
func (p *PyPiExtractor) GetEcosystem() packagev1.Ecosystem {
|
||||
return packagev1.Ecosystem_ECOSYSTEM_PYPI
|
||||
}
|
||||
|
||||
func (p *PyPiExtractor) Extract(depFile lockfile.DepFile, filename string) ([]lockfile.PackageDetails, error) {
|
||||
switch filename {
|
||||
case "requirements.txt":
|
||||
extractor := lockfile.RequirementsTxtExtractor{}
|
||||
return extractor.Extract(depFile)
|
||||
case "Pipfile.lock":
|
||||
extractor := lockfile.PipenvLockExtractor{}
|
||||
return extractor.Extract(depFile)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported PyPI lockfile: %s", filename)
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractorManager manages all ecosystem extractors
|
||||
type ExtractorManager struct {
|
||||
extractors map[packagev1.Ecosystem]EcosystemExtractor
|
||||
extractors map[PackageManagerName]PackageManagerExtractor
|
||||
}
|
||||
|
||||
func NewExtractorManager() *ExtractorManager {
|
||||
return &ExtractorManager{
|
||||
extractors: map[packagev1.Ecosystem]EcosystemExtractor{
|
||||
packagev1.Ecosystem_ECOSYSTEM_NPM: &NpmExtractor{},
|
||||
packagev1.Ecosystem_ECOSYSTEM_PYPI: &PyPiExtractor{},
|
||||
extractors: map[PackageManagerName]PackageManagerExtractor{
|
||||
Npm: &NpmExtractor{},
|
||||
Pnpm: &PnpmExtractor{},
|
||||
Pip: &PipExtractor{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (em *ExtractorManager) GetExtractorForEcosystem(ecosystem packagev1.Ecosystem) (EcosystemExtractor, bool) {
|
||||
extractor, exists := em.extractors[ecosystem]
|
||||
return extractor, exists
|
||||
func (e *ExtractorManager) GetExtractorForPackageManager(pmn PackageManagerName) PackageManagerExtractor {
|
||||
return e.extractors[pmn]
|
||||
}
|
||||
|
||||
func (em *ExtractorManager) GetSupportedFilesForEcosystem(ecosystem packagev1.Ecosystem) []string {
|
||||
if extractor, exists := em.extractors[ecosystem]; exists {
|
||||
return extractor.GetSupportedFiles()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (em *ExtractorManager) GetAllSupportedFiles() []string {
|
||||
var allFiles []string
|
||||
for _, extractor := range em.extractors {
|
||||
allFiles = append(allFiles, extractor.GetSupportedFiles()...)
|
||||
}
|
||||
return allFiles
|
||||
func (e *ExtractorManager) GetSupportedFilesForPackageManager(pmn PackageManagerName) []string {
|
||||
return e.extractors[pmn].GetSupportedFiles()
|
||||
}
|
||||
|
||||
+17
-99
@@ -4,23 +4,19 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
|
||||
"github.com/google/osv-scanner/pkg/lockfile"
|
||||
"github.com/safedep/dry/log"
|
||||
)
|
||||
|
||||
type ExtractorConfig struct {
|
||||
// List of extractors used by an ecosystem
|
||||
ExtractorsName []string
|
||||
ExtractorEcosystem packagev1.Ecosystem
|
||||
ScanDir string
|
||||
ExtractorPackageManager PackageManagerName
|
||||
ScanDir string
|
||||
}
|
||||
|
||||
type extractor struct {
|
||||
Config ExtractorConfig
|
||||
extractorManager *ExtractorManager
|
||||
extractorManager ExtractorManager
|
||||
}
|
||||
|
||||
func NewDefaultExtractorConfig() *ExtractorConfig {
|
||||
@@ -32,7 +28,7 @@ func NewDefaultExtractorConfig() *ExtractorConfig {
|
||||
func New(config ExtractorConfig) *extractor {
|
||||
return &extractor{
|
||||
Config: config,
|
||||
extractorManager: NewExtractorManager(),
|
||||
extractorManager: *NewExtractorManager(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,8 +46,14 @@ func (e *extractor) ExtractManifest() ([]*packagev1.PackageVersion, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
extractor, err := e.getExtractorForFile()
|
||||
if err != nil {
|
||||
log.Warnf("failed to get extractor\n")
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract packages from this lockfile
|
||||
packages, err := e.extractFromLockfile(filePath)
|
||||
packages, err := extractor.Extract(filePath, e.Config.ScanDir)
|
||||
if err != nil {
|
||||
log.Warnf("failed to extract from %s: %v\n", filePath, err)
|
||||
continue
|
||||
@@ -64,97 +66,13 @@ func (e *extractor) ExtractManifest() ([]*packagev1.PackageVersion, error) {
|
||||
}
|
||||
|
||||
func (e *extractor) getFilesToCheck() []string {
|
||||
// If specific extractors are configured, use those
|
||||
if len(e.Config.ExtractorsName) > 0 {
|
||||
return e.Config.ExtractorsName
|
||||
}
|
||||
|
||||
// If ecosystem is specified, return files for that ecosystem
|
||||
if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
|
||||
return e.extractorManager.GetSupportedFilesForEcosystem(e.Config.ExtractorEcosystem)
|
||||
}
|
||||
|
||||
// If no ecosystem specified, check all known extractors
|
||||
return e.extractorManager.GetAllSupportedFiles()
|
||||
return e.extractorManager.GetSupportedFilesForPackageManager(e.Config.ExtractorPackageManager)
|
||||
}
|
||||
|
||||
func (e *extractor) extractFromLockfile(path string) ([]*packagev1.PackageVersion, error) {
|
||||
filename := filepath.Base(path)
|
||||
|
||||
// Find the appropriate extractor for this file
|
||||
ecosystemExtractor, ecosystem, err := e.getExtractorForFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func (e *extractor) getExtractorForFile() (PackageManagerExtractor, error) {
|
||||
extractor := e.extractorManager.GetExtractorForPackageManager(e.Config.ExtractorPackageManager)
|
||||
if extractor == nil {
|
||||
return nil, fmt.Errorf("no extractor found for package manager: %s", e.Config.ExtractorPackageManager)
|
||||
}
|
||||
|
||||
// Open the file as a DepFile for extraction
|
||||
depFile, err := lockfile.OpenLocalDepFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file %s: %w", path, err)
|
||||
}
|
||||
defer depFile.Close()
|
||||
|
||||
// Extract dependencies using the appropriate extractor
|
||||
deps, err := ecosystemExtractor.Extract(depFile, filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract from %s: %w", path, err)
|
||||
}
|
||||
|
||||
packages := e.convertToPackageVersions(deps, ecosystem)
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
func (e *extractor) getExtractorForFile(filename string) (EcosystemExtractor, packagev1.Ecosystem, error) {
|
||||
// If ecosystem is configured, use that directly
|
||||
if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
|
||||
extractor, exists := e.extractorManager.GetExtractorForEcosystem(e.Config.ExtractorEcosystem)
|
||||
if !exists {
|
||||
return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("no extractor found for ecosystem: %v", e.Config.ExtractorEcosystem)
|
||||
}
|
||||
|
||||
// Verify the file is supported by this ecosystem
|
||||
supportedFiles := extractor.GetSupportedFiles()
|
||||
for _, supportedFile := range supportedFiles {
|
||||
if filename == supportedFile || strings.Contains(filename, supportedFile) {
|
||||
return extractor, e.Config.ExtractorEcosystem, nil
|
||||
}
|
||||
}
|
||||
return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("file %s not supported by ecosystem %v", filename, e.Config.ExtractorEcosystem)
|
||||
}
|
||||
|
||||
// Try to find extractor by filename
|
||||
for ecosystem, extractor := range e.extractorManager.extractors {
|
||||
supportedFiles := extractor.GetSupportedFiles()
|
||||
for _, supportedFile := range supportedFiles {
|
||||
if filename == supportedFile || strings.Contains(filename, supportedFile) {
|
||||
return extractor, ecosystem, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, fmt.Errorf("unsupported lockfile type: %s", filename)
|
||||
}
|
||||
|
||||
func (e *extractor) convertToPackageVersions(deps []lockfile.PackageDetails, ecosystem packagev1.Ecosystem) []*packagev1.PackageVersion {
|
||||
var packages []*packagev1.PackageVersion
|
||||
|
||||
for _, dep := range deps {
|
||||
// Use the configured ecosystem if provided, otherwise use the detected one
|
||||
targetEcosystem := ecosystem
|
||||
if e.Config.ExtractorEcosystem != packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED {
|
||||
targetEcosystem = e.Config.ExtractorEcosystem
|
||||
}
|
||||
|
||||
packageVer := &packagev1.PackageVersion{
|
||||
Package: &packagev1.Package{
|
||||
Name: dep.Name,
|
||||
Ecosystem: targetEcosystem,
|
||||
},
|
||||
Version: dep.Version,
|
||||
}
|
||||
|
||||
packages = append(packages, packageVer)
|
||||
}
|
||||
|
||||
return packages
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
|
||||
"github.com/google/osv-scalibr/extractor/filesystem"
|
||||
"github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson"
|
||||
"github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock"
|
||||
"github.com/google/osv-scalibr/fs"
|
||||
)
|
||||
|
||||
// NpmExtractor handles package-lock.json files
|
||||
type NpmExtractor struct{}
|
||||
|
||||
func (n *NpmExtractor) GetSupportedFiles() []string {
|
||||
return []string{"package-lock.json"}
|
||||
}
|
||||
|
||||
func (n *NpmExtractor) GetEcosystem() packagev1.Ecosystem {
|
||||
return packagev1.Ecosystem_ECOSYSTEM_NPM
|
||||
}
|
||||
|
||||
func (n *NpmExtractor) GetPackageManager() PackageManagerName {
|
||||
return Npm
|
||||
}
|
||||
|
||||
func (n *NpmExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
return parseNpmPackageLockFile(lockfilePath, scanDir)
|
||||
}
|
||||
|
||||
func parseNpmPackageLockFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
packagelockExtractor := packagelockjson.NewDefault()
|
||||
|
||||
file, err := os.Open(lockfilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open lockfile: %s", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
inputConfig := &filesystem.ScanInput{
|
||||
FS: fs.DirFS(scanDir),
|
||||
Path: lockfilePath,
|
||||
Reader: file,
|
||||
}
|
||||
|
||||
inventory, err := packagelockExtractor.Extract(context.Background(), inputConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract packages: %s", err)
|
||||
}
|
||||
|
||||
var packages []*packagev1.PackageVersion
|
||||
|
||||
for _, invPkg := range inventory.Packages {
|
||||
pkg := &packagev1.PackageVersion{
|
||||
Package: &packagev1.Package{
|
||||
Name: invPkg.Name,
|
||||
Ecosystem: packagev1.Ecosystem_ECOSYSTEM_NPM,
|
||||
},
|
||||
Version: invPkg.Version,
|
||||
}
|
||||
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
// PnpmExtractor handles pnpm-lock.yaml files
|
||||
type PnpmExtractor struct{}
|
||||
|
||||
func (p *PnpmExtractor) GetSupportedFiles() []string {
|
||||
return []string{"pnpm-lock.yaml"}
|
||||
}
|
||||
|
||||
func (p *PnpmExtractor) GetEcosystem() packagev1.Ecosystem {
|
||||
return packagev1.Ecosystem_ECOSYSTEM_NPM
|
||||
}
|
||||
|
||||
func (p *PnpmExtractor) GetPackageManager() PackageManagerName {
|
||||
return Pnpm
|
||||
}
|
||||
|
||||
func (p *PnpmExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
return parsePnpmLockFile(lockfilePath, scanDir)
|
||||
}
|
||||
|
||||
func parsePnpmLockFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
pnpmLockExtractor := pnpmlock.New()
|
||||
|
||||
file, err := os.Open(lockfilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open lockfile: %s", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
inputConfig := &filesystem.ScanInput{
|
||||
FS: fs.DirFS(scanDir),
|
||||
Path: lockfilePath,
|
||||
Reader: file,
|
||||
}
|
||||
|
||||
inventory, err := pnpmLockExtractor.Extract(context.Background(), inputConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract packages: %s", err)
|
||||
}
|
||||
|
||||
var packages []*packagev1.PackageVersion
|
||||
|
||||
for _, invPkg := range inventory.Packages {
|
||||
pkg := &packagev1.PackageVersion{
|
||||
Package: &packagev1.Package{
|
||||
Name: invPkg.Name,
|
||||
Ecosystem: packagev1.Ecosystem_ECOSYSTEM_NPM,
|
||||
},
|
||||
Version: invPkg.Version,
|
||||
}
|
||||
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
|
||||
"github.com/google/osv-scalibr/extractor/filesystem"
|
||||
"github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements"
|
||||
"github.com/google/osv-scalibr/fs"
|
||||
)
|
||||
|
||||
// PipExtractor handles requirements.txt files
|
||||
type PipExtractor struct{}
|
||||
|
||||
func (p *PipExtractor) GetSupportedFiles() []string {
|
||||
return []string{"requirements.txt"}
|
||||
}
|
||||
|
||||
func (p *PipExtractor) GetEcosystem() packagev1.Ecosystem {
|
||||
return packagev1.Ecosystem_ECOSYSTEM_PYPI
|
||||
}
|
||||
|
||||
func (p *PipExtractor) GetPackageManager() PackageManagerName {
|
||||
return Pip
|
||||
}
|
||||
|
||||
func (n *PipExtractor) Extract(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
return parseRequirementsTxtFile(lockfilePath, scanDir)
|
||||
}
|
||||
|
||||
func parseRequirementsTxtFile(lockfilePath, scanDir string) ([]*packagev1.PackageVersion, error) {
|
||||
requirementsExtractor := requirements.NewDefault()
|
||||
|
||||
file, err := os.Open(lockfilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open lockfile: %s", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
inputConfig := &filesystem.ScanInput{
|
||||
FS: fs.DirFS(scanDir),
|
||||
Path: lockfilePath,
|
||||
Reader: file,
|
||||
}
|
||||
|
||||
inventory, err := requirementsExtractor.Extract(context.Background(), inputConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract packages: %s", err)
|
||||
}
|
||||
|
||||
var packages []*packagev1.PackageVersion
|
||||
|
||||
for _, invPkg := range inventory.Packages {
|
||||
pkg := &packagev1.PackageVersion{
|
||||
Package: &packagev1.Package{
|
||||
Name: invPkg.Name,
|
||||
Ecosystem: packagev1.Ecosystem_ECOSYSTEM_PYPI,
|
||||
},
|
||||
Version: invPkg.Version,
|
||||
}
|
||||
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
Reference in New Issue
Block a user