mirror of
https://github.com/safedep/pmg.git
synced 2026-08-03 07:24:09 +02:00
Add proxy support for pypi package managers (#150)
* initial pypi registry implementation * support proxy mode for pypi package managers * support proxy mode for pypi package managers - 2 * rm default mode as proxy for pip3 * update goproxy version & fix pypi proxy failing on 304 * add PIP_RETRIES=0 env * update pmg e2e & add proxy mode e2e for pypi * rm safedep-test-pkg for pypi proxy e2e
This commit is contained in:
Sahil Bansal
@@ -43,6 +43,14 @@ func (f *InterceptorFactory) CreateInterceptor(ecosystem packagev1.Ecosystem) (p
|
||||
f.confirmationChan,
|
||||
), nil
|
||||
|
||||
case packagev1.Ecosystem_ECOSYSTEM_PYPI:
|
||||
return NewPypiRegistryInterceptor(
|
||||
f.analyzer,
|
||||
f.cache,
|
||||
f.statsCollector,
|
||||
f.confirmationChan,
|
||||
), nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("proxy-based interception not yet supported for ecosystem: %s", ecosystem.String())
|
||||
}
|
||||
@@ -52,6 +60,7 @@ func (f *InterceptorFactory) CreateInterceptor(ecosystem packagev1.Ecosystem) (p
|
||||
func SupportedEcosystems() []packagev1.Ecosystem {
|
||||
return []packagev1.Ecosystem{
|
||||
packagev1.Ecosystem_ECOSYSTEM_NPM,
|
||||
packagev1.Ecosystem_ECOSYSTEM_PYPI,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package interceptors
|
||||
|
||||
import (
|
||||
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
|
||||
"github.com/safedep/dry/log"
|
||||
"github.com/safedep/pmg/analyzer"
|
||||
"github.com/safedep/pmg/proxy"
|
||||
)
|
||||
|
||||
var pypiRegistryDomains = registryConfigMap{
|
||||
"files.pythonhosted.org": {
|
||||
Host: "files.pythonhosted.org",
|
||||
SupportedForAnalysis: true,
|
||||
Parser: pypiFilesParser{},
|
||||
},
|
||||
"pypi.org": {
|
||||
Host: "pypi.org",
|
||||
SupportedForAnalysis: true,
|
||||
Parser: pypiOrgParser{},
|
||||
},
|
||||
// Test PyPI instance
|
||||
"test.pypi.org": {
|
||||
Host: "test.pypi.org",
|
||||
SupportedForAnalysis: false, // Skip analysis for test PyPI
|
||||
Parser: pypiOrgParser{},
|
||||
},
|
||||
"test-files.pythonhosted.org": {
|
||||
Host: "test-files.pythonhosted.org",
|
||||
SupportedForAnalysis: false, // Skip analysis for test PyPI files
|
||||
Parser: pypiFilesParser{},
|
||||
},
|
||||
}
|
||||
|
||||
// PypiRegistryInterceptor intercepts PyPI registry requests and analyzes packages for malware
|
||||
// It embeds baseRegistryInterceptor to reuse ecosystem agnostic functionality
|
||||
type PypiRegistryInterceptor struct {
|
||||
baseRegistryInterceptor
|
||||
}
|
||||
|
||||
var _ proxy.Interceptor = (*PypiRegistryInterceptor)(nil)
|
||||
|
||||
// NewPypiRegistryInterceptor creates a new PyPI registry interceptor
|
||||
func NewPypiRegistryInterceptor(
|
||||
analyzer analyzer.PackageVersionAnalyzer,
|
||||
cache AnalysisCache,
|
||||
statsCollector *AnalysisStatsCollector,
|
||||
confirmationChan chan *ConfirmationRequest,
|
||||
) *PypiRegistryInterceptor {
|
||||
return &PypiRegistryInterceptor{
|
||||
baseRegistryInterceptor: baseRegistryInterceptor{
|
||||
analyzer: analyzer,
|
||||
cache: cache,
|
||||
statsCollector: statsCollector,
|
||||
confirmationChan: confirmationChan,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the interceptor name for logging
|
||||
func (i *PypiRegistryInterceptor) Name() string {
|
||||
return "pypi-registry-interceptor"
|
||||
}
|
||||
|
||||
// ShouldIntercept determines if this interceptor should handle the given request
|
||||
func (i *PypiRegistryInterceptor) ShouldIntercept(ctx *proxy.RequestContext) bool {
|
||||
return pypiRegistryDomains.ContainsHostname(ctx.Hostname)
|
||||
}
|
||||
|
||||
// HandleRequest processes the request and returns response action
|
||||
// We take a fail-open approach here, allowing requests that we can't parse the package information from the URL.
|
||||
func (i *PypiRegistryInterceptor) HandleRequest(ctx *proxy.RequestContext) (*proxy.InterceptorResponse, error) {
|
||||
log.Debugf("[%s] Handling PyPI registry request: %s", ctx.RequestID, ctx.URL.Path)
|
||||
|
||||
// Get registry configuration
|
||||
config := pypiRegistryDomains.GetConfigForHostname(ctx.Hostname)
|
||||
if config == nil {
|
||||
// Shouldn't happen if ShouldIntercept is working correctly
|
||||
log.Warnf("[%s] No registry config found for hostname: %s", ctx.RequestID, ctx.Hostname)
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
// Skip analysis for registries that are not supported for analysis
|
||||
if !config.SupportedForAnalysis {
|
||||
log.Debugf("[%s] Skipping analysis for %s registry (not supported for analysis): %s",
|
||||
ctx.RequestID, config.Host, ctx.URL.String())
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
// Parse URL using registry-specific strategy
|
||||
pkgInfo, err := config.Parser.ParseURL(ctx.URL.Path)
|
||||
if err != nil {
|
||||
log.Warnf("[%s] Failed to parse PyPI registry URL %s for %s: %v",
|
||||
ctx.RequestID, ctx.URL.Path, config.Host, err)
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
// Only analyze actual file downloads (sdist or wheel)
|
||||
// Metadata requests (Simple API or JSON API) are allowed through
|
||||
if !pkgInfo.IsFileDownload() {
|
||||
log.Debugf("[%s] Skipping analysis for metadata request: %s", ctx.RequestID, pkgInfo.GetName())
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
// Ensure we have both name and version for analysis
|
||||
if pkgInfo.GetName() == "" || pkgInfo.GetVersion() == "" {
|
||||
log.Warnf("[%s] Incomplete package info from URL %s: name=%s, version=%s",
|
||||
ctx.RequestID, ctx.URL.Path, pkgInfo.GetName(), pkgInfo.GetVersion())
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
// Get file type for logging if available
|
||||
fileType := ""
|
||||
if pypiInfo, ok := pkgInfo.(*pypiPackageInfo); ok {
|
||||
fileType = pypiInfo.FileType()
|
||||
}
|
||||
log.Debugf("[%s] Analyzing PyPI package: %s@%s (type: %s)",
|
||||
ctx.RequestID, pkgInfo.GetName(), pkgInfo.GetVersion(), fileType)
|
||||
|
||||
result, err := i.analyzePackage(
|
||||
ctx,
|
||||
packagev1.Ecosystem_ECOSYSTEM_PYPI,
|
||||
pkgInfo.GetName(),
|
||||
pkgInfo.GetVersion(),
|
||||
)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] Failed to analyze package %s@%s: %v", ctx.RequestID, pkgInfo.GetName(), pkgInfo.GetVersion(), err)
|
||||
return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil
|
||||
}
|
||||
|
||||
return i.handleAnalysisResult(ctx, packagev1.Ecosystem_ECOSYSTEM_PYPI, pkgInfo.GetName(), pkgInfo.GetVersion(), result)
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
package interceptors
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// pypiPackageInfo represents parsed package information from a PyPI registry URL
|
||||
type pypiPackageInfo struct {
|
||||
name string
|
||||
version string
|
||||
isDownload bool // True if this is a file download (sdist or wheel)
|
||||
fileType string // "sdist", "wheel", or empty for non-download requests
|
||||
}
|
||||
|
||||
// Ensure pypiPackageInfo implements packageInfo interface
|
||||
var _ packageInfo = (*pypiPackageInfo)(nil)
|
||||
|
||||
// GetName returns the package name
|
||||
func (p *pypiPackageInfo) GetName() string {
|
||||
return p.name
|
||||
}
|
||||
|
||||
// GetVersion returns the package version
|
||||
func (p *pypiPackageInfo) GetVersion() string {
|
||||
return p.version
|
||||
}
|
||||
|
||||
// IsFileDownload returns true if this is a file download (sdist or wheel)
|
||||
func (p *pypiPackageInfo) IsFileDownload() bool {
|
||||
return p.isDownload
|
||||
}
|
||||
|
||||
// FileType returns the file type ("sdist", "wheel", or empty)
|
||||
func (p *pypiPackageInfo) FileType() string {
|
||||
return p.fileType
|
||||
}
|
||||
|
||||
// pypiFilesParser parses URLs from files.pythonhosted.org
|
||||
// This is where PyPI serves package files (sdists and wheels)
|
||||
type pypiFilesParser struct{}
|
||||
|
||||
// Ensure pypiFilesParser implements RegistryURLParser interface
|
||||
var _ registryURLParser = pypiFilesParser{}
|
||||
|
||||
// ParseURL parses files.pythonhosted.org URL paths
|
||||
// URL patterns:
|
||||
// - /packages/{hash_dirs}/{filename}
|
||||
// Where filename can be:
|
||||
// - {name}-{version}.tar.gz (sdist)
|
||||
// - {name}-{version}.zip (sdist)
|
||||
// - {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl (wheel)
|
||||
func (p pypiFilesParser) ParseURL(urlPath string) (packageInfo, error) {
|
||||
// Remove leading and trailing slashes
|
||||
urlPath = strings.Trim(urlPath, "/")
|
||||
|
||||
if urlPath == "" {
|
||||
return nil, fmt.Errorf("empty URL path")
|
||||
}
|
||||
|
||||
// Split path into segments
|
||||
segments := strings.Split(urlPath, "/")
|
||||
|
||||
// files.pythonhosted.org paths start with "packages"
|
||||
// Format: packages/{hash_prefix}/{filename}
|
||||
// The hash prefix can be variable length (typically 2-3 directory levels)
|
||||
if len(segments) < 2 {
|
||||
return nil, fmt.Errorf("invalid PyPI files URL: not enough segments")
|
||||
}
|
||||
|
||||
// The filename is always the last segment
|
||||
filename := segments[len(segments)-1]
|
||||
|
||||
// Check if it's a packages download path
|
||||
if segments[0] != "packages" {
|
||||
return nil, fmt.Errorf("invalid PyPI files URL: expected 'packages' prefix, got %s", segments[0])
|
||||
}
|
||||
|
||||
return parseFilename(filename)
|
||||
}
|
||||
|
||||
// pypiOrgParser parses URLs from pypi.org (Simple API and JSON API)
|
||||
type pypiOrgParser struct{}
|
||||
|
||||
// Ensure pypiOrgParser implements RegistryURLParser interface
|
||||
var _ registryURLParser = pypiOrgParser{}
|
||||
|
||||
// ParseURL parses pypi.org URL paths
|
||||
// URL patterns:
|
||||
// - /simple/{package}/ (Simple API - package index)
|
||||
// - /simple/{package}/{filename} (Simple API - file redirect, rare)
|
||||
// - /pypi/{package}/json (JSON API - package metadata)
|
||||
// - /pypi/{package}/{version}/json (JSON API - version metadata)
|
||||
func (p pypiOrgParser) ParseURL(urlPath string) (packageInfo, error) {
|
||||
// Remove leading and trailing slashes
|
||||
urlPath = strings.Trim(urlPath, "/")
|
||||
|
||||
if urlPath == "" {
|
||||
return nil, fmt.Errorf("empty URL path")
|
||||
}
|
||||
|
||||
// Split path into segments
|
||||
segments := strings.Split(urlPath, "/")
|
||||
|
||||
if len(segments) < 2 {
|
||||
return nil, fmt.Errorf("invalid pypi.org URL: not enough segments")
|
||||
}
|
||||
|
||||
switch segments[0] {
|
||||
case "simple":
|
||||
// Simple API: /simple/{package}/ or /simple/{package}/{filename}
|
||||
return parseSimpleAPIURL(segments[1:])
|
||||
case "pypi":
|
||||
// JSON API: /pypi/{package}/json or /pypi/{package}/{version}/json
|
||||
return parseJSONAPIURL(segments[1:])
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown pypi.org path prefix: %s", segments[0])
|
||||
}
|
||||
}
|
||||
|
||||
// parseSimpleAPIURL parses Simple API URL paths
|
||||
func parseSimpleAPIURL(segments []string) (*pypiPackageInfo, error) {
|
||||
if len(segments) == 0 {
|
||||
return nil, fmt.Errorf("invalid Simple API URL: missing package name")
|
||||
}
|
||||
|
||||
packageName := segments[0]
|
||||
|
||||
// Simple API index request: /simple/{package}/
|
||||
if len(segments) == 1 {
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(packageName),
|
||||
isDownload: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Simple API might include filename (for redirects): /simple/{package}/{filename}
|
||||
if len(segments) == 2 {
|
||||
filename := segments[1]
|
||||
info, err := parseFilename(filename)
|
||||
if err != nil {
|
||||
// If we can't parse the filename, treat it as a non-download request
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(packageName),
|
||||
isDownload: false,
|
||||
}, nil
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("invalid Simple API URL format: too many segments")
|
||||
}
|
||||
|
||||
// parseJSONAPIURL parses JSON API URL paths
|
||||
func parseJSONAPIURL(segments []string) (*pypiPackageInfo, error) {
|
||||
if len(segments) == 0 {
|
||||
return nil, fmt.Errorf("invalid JSON API URL: missing package name")
|
||||
}
|
||||
|
||||
packageName := segments[0]
|
||||
|
||||
// /pypi/{package}/json - package metadata (no specific version)
|
||||
if len(segments) == 2 && segments[1] == "json" {
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(packageName),
|
||||
isDownload: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// /pypi/{package}/{version}/json - version metadata
|
||||
if len(segments) == 3 && segments[2] == "json" {
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(packageName),
|
||||
version: segments[1],
|
||||
isDownload: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("invalid JSON API URL format")
|
||||
}
|
||||
|
||||
// parseFilename extracts package name and version from a PyPI distribution filename
|
||||
func parseFilename(filename string) (*pypiPackageInfo, error) {
|
||||
// Try to parse as wheel first
|
||||
if strings.HasSuffix(filename, ".whl") {
|
||||
return parseWheelFilename(filename)
|
||||
}
|
||||
|
||||
// Try to parse as sdist (tar.gz or zip)
|
||||
if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".zip") {
|
||||
return parseSdistFilename(filename)
|
||||
}
|
||||
|
||||
// Check for other archive formats that PyPI might serve
|
||||
if strings.HasSuffix(filename, ".tar.bz2") || strings.HasSuffix(filename, ".tgz") {
|
||||
return parseSdistFilename(filename)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported file type: %s", filename)
|
||||
}
|
||||
|
||||
// parseWheelFilename parses a wheel filename to extract package info
|
||||
// Wheel filename format: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
|
||||
// Examples:
|
||||
// - requests-2.28.0-py3-none-any.whl
|
||||
// - numpy-1.24.0-cp311-cp311-linux_x86_64.whl
|
||||
// - package_name-1.0.0-1-py3-none-any.whl (with build tag)
|
||||
func parseWheelFilename(filename string) (*pypiPackageInfo, error) {
|
||||
// Remove .whl extension
|
||||
basename := strings.TrimSuffix(filename, ".whl")
|
||||
|
||||
// Split by '-' to get components
|
||||
// Minimum: name-version-python-abi-platform (5 parts)
|
||||
// With build tag: name-version-build-python-abi-platform (6 parts)
|
||||
parts := strings.Split(basename, "-")
|
||||
|
||||
if len(parts) < 5 {
|
||||
return nil, fmt.Errorf("invalid wheel filename: not enough components in %s", filename)
|
||||
}
|
||||
|
||||
// The last 3 parts are always: python_tag, abi_tag, platform_tag
|
||||
// Before that is either: name, version OR name, version, build_tag
|
||||
// We need to find where the version is
|
||||
|
||||
// Work backwards: last 3 are tags
|
||||
// If 6+ parts, could have build tag
|
||||
// If 5 parts, no build tag
|
||||
|
||||
var name, version string
|
||||
|
||||
if len(parts) == 5 {
|
||||
// name-version-python-abi-platform
|
||||
name = parts[0]
|
||||
version = parts[1]
|
||||
} else if len(parts) == 6 {
|
||||
// Could be:
|
||||
// - name-version-build-python-abi-platform (6 parts, with build tag)
|
||||
// - name_with_underscore-version-python-abi-platform (can't be this, underscores in names are normalized)
|
||||
// Build tags are numeric (PEP 427)
|
||||
if isBuildTag(parts[2]) {
|
||||
name = parts[0]
|
||||
version = parts[1]
|
||||
} else {
|
||||
// The name might contain a hyphen that wasn't normalized
|
||||
// This shouldn't happen with properly normalized names, but handle it
|
||||
name = parts[0] + "_" + parts[1]
|
||||
version = parts[2]
|
||||
}
|
||||
} else {
|
||||
// More than 6 parts - name contains hyphens or there's a build tag
|
||||
// Try to find version by looking for semver-like pattern
|
||||
name, version = extractNameVersionFromParts(parts[:len(parts)-3])
|
||||
if name == "" || version == "" {
|
||||
return nil, fmt.Errorf("could not parse wheel filename: %s", filename)
|
||||
}
|
||||
}
|
||||
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(name),
|
||||
version: version,
|
||||
isDownload: true,
|
||||
fileType: "wheel",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// isBuildTag checks if a string looks like a wheel build tag (numeric)
|
||||
func isBuildTag(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseSdistFilename parses a source distribution filename to extract package info
|
||||
// Sdist filename format: {name}-{version}.tar.gz or {name}-{version}.zip
|
||||
// Examples:
|
||||
// - requests-2.28.0.tar.gz
|
||||
// - Flask-RESTful-0.3.10.tar.gz (note: hyphens in name)
|
||||
func parseSdistFilename(filename string) (*pypiPackageInfo, error) {
|
||||
// Remove extension
|
||||
basename := filename
|
||||
for _, ext := range []string{".tar.gz", ".tar.bz2", ".tgz", ".zip"} {
|
||||
if strings.HasSuffix(basename, ext) {
|
||||
basename = strings.TrimSuffix(basename, ext)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Find the version by looking for the last hyphen followed by a version-like string
|
||||
// This is tricky because package names can contain hyphens
|
||||
name, version := extractNameVersionFromSdist(basename)
|
||||
if name == "" || version == "" {
|
||||
return nil, fmt.Errorf("could not parse sdist filename: %s", filename)
|
||||
}
|
||||
|
||||
return &pypiPackageInfo{
|
||||
name: denormalizePyPIPackageName(name),
|
||||
version: version,
|
||||
isDownload: true,
|
||||
fileType: "sdist",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractNameVersionFromSdist extracts name and version from a sdist basename
|
||||
// The challenge is that package names can contain hyphens, so we need to find
|
||||
// where the name ends and the version begins
|
||||
func extractNameVersionFromSdist(basename string) (string, string) {
|
||||
// Version pattern: starts with a digit, may contain digits, dots, and pre-release suffixes
|
||||
versionPattern := regexp.MustCompile(`^\d+(\.\d+)*([._-]?(a|alpha|b|beta|c|rc|pre|post|dev|final)\.?\d*)*(\+[a-zA-Z0-9._-]+)?$`)
|
||||
|
||||
// Split by hyphen and try to find where version starts
|
||||
parts := strings.Split(basename, "-")
|
||||
|
||||
// Try from the end, looking for version-like parts
|
||||
for i := len(parts) - 1; i > 0; i-- {
|
||||
potentialVersion := strings.Join(parts[i:], "-")
|
||||
// Check if this could be a version
|
||||
if versionPattern.MatchString(potentialVersion) {
|
||||
name := strings.Join(parts[:i], "-")
|
||||
return name, potentialVersion
|
||||
}
|
||||
|
||||
// Also try just the single part as version
|
||||
if versionPattern.MatchString(parts[i]) {
|
||||
name := strings.Join(parts[:i], "-")
|
||||
return name, parts[i]
|
||||
}
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// extractNameVersionFromParts extracts name and version from wheel filename parts
|
||||
// (excluding the python-abi-platform tags)
|
||||
func extractNameVersionFromParts(parts []string) (string, string) {
|
||||
if len(parts) < 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// Version pattern for wheels
|
||||
versionPattern := regexp.MustCompile(`^\d+(\.\d+)*([._]?(a|alpha|b|beta|c|rc|pre|post|dev|final)\d*)*(\+[a-zA-Z0-9._]+)?$`)
|
||||
|
||||
// Try from the end, looking for version-like parts
|
||||
for i := len(parts) - 1; i > 0; i-- {
|
||||
if versionPattern.MatchString(parts[i]) {
|
||||
// Check if next part is a build tag (numeric only)
|
||||
if i+1 < len(parts) && isBuildTag(parts[i+1]) {
|
||||
// This is the version, parts[i+1] is build tag
|
||||
name := strings.Join(parts[:i], "_")
|
||||
return name, parts[i]
|
||||
}
|
||||
name := strings.Join(parts[:i], "_")
|
||||
return name, parts[i]
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: assume first part is name, second is version
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
|
||||
// denormalizePyPIPackageName converts a normalized package name back to a more canonical form
|
||||
// PyPI normalizes names by replacing [-_.] with - and lowercasing
|
||||
// We can't fully reverse this, but we keep the normalized form which works for lookups
|
||||
func denormalizePyPIPackageName(name string) string {
|
||||
// Convert underscores to hyphens (common PyPI convention)
|
||||
// Keep lowercase as that's the normalized form
|
||||
return strings.ReplaceAll(strings.ToLower(name), "_", "-")
|
||||
}
|
||||
@@ -0,0 +1,590 @@
|
||||
package interceptors
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestPypiFilesParser_ParseURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
urlPath string
|
||||
wantName string
|
||||
wantVersion string
|
||||
wantIsDownload bool
|
||||
wantFileType string
|
||||
wantErr bool
|
||||
}{
|
||||
// Source distributions (sdist)
|
||||
{
|
||||
name: "sdist tar.gz simple package",
|
||||
urlPath: "/packages/ab/cd/abcd1234/requests-2.28.0.tar.gz",
|
||||
wantName: "requests",
|
||||
wantVersion: "2.28.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist tar.gz with hyphenated name",
|
||||
urlPath: "/packages/12/34/5678abcd/Flask-RESTful-0.3.10.tar.gz",
|
||||
wantName: "flask-restful",
|
||||
wantVersion: "0.3.10",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist zip format",
|
||||
urlPath: "/packages/aa/bb/ccdd/some-package-1.0.0.zip",
|
||||
wantName: "some-package",
|
||||
wantVersion: "1.0.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist with prerelease version",
|
||||
urlPath: "/packages/ff/ee/ddcc/mypackage-2.0.0rc1.tar.gz",
|
||||
wantName: "mypackage",
|
||||
wantVersion: "2.0.0rc1",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist with dev version",
|
||||
urlPath: "/packages/11/22/3344/testpkg-0.1.0.dev1.tar.gz",
|
||||
wantName: "testpkg",
|
||||
wantVersion: "0.1.0.dev1",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist with post version",
|
||||
urlPath: "/packages/aa/bb/cc/package-1.0.0.post1.tar.gz",
|
||||
wantName: "package",
|
||||
wantVersion: "1.0.0.post1",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sdist with local version identifier",
|
||||
urlPath: "/packages/dd/ee/ff/mylib-1.2.3+local.tar.gz",
|
||||
wantName: "mylib",
|
||||
wantVersion: "1.2.3+local",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
// Wheel files
|
||||
{
|
||||
name: "wheel simple package",
|
||||
urlPath: "/packages/ab/cd/ef12/requests-2.28.0-py3-none-any.whl",
|
||||
wantName: "requests",
|
||||
wantVersion: "2.28.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with platform-specific tags",
|
||||
urlPath: "/packages/12/34/56/numpy-1.24.0-cp311-cp311-linux_x86_64.whl",
|
||||
wantName: "numpy",
|
||||
wantVersion: "1.24.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with manylinux platform",
|
||||
urlPath: "/packages/aa/bb/cc/cryptography-41.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
|
||||
wantName: "cryptography",
|
||||
wantVersion: "41.0.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with underscore in name (normalized)",
|
||||
urlPath: "/packages/11/22/33/some_package-1.0.0-py3-none-any.whl",
|
||||
wantName: "some-package",
|
||||
wantVersion: "1.0.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with build tag",
|
||||
urlPath: "/packages/ff/ee/dd/mypackage-1.0.0-1-py3-none-any.whl",
|
||||
wantName: "mypackage",
|
||||
wantVersion: "1.0.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel windows platform",
|
||||
urlPath: "/packages/aa/bb/cc/pywin32-306-cp311-cp311-win_amd64.whl",
|
||||
wantName: "pywin32",
|
||||
wantVersion: "306",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel macos platform",
|
||||
urlPath: "/packages/dd/ee/ff/tensorflow-2.15.0-cp311-cp311-macosx_10_15_x86_64.whl",
|
||||
wantName: "tensorflow",
|
||||
wantVersion: "2.15.0",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
// Real-world examples
|
||||
{
|
||||
name: "real django sdist",
|
||||
urlPath: "/packages/b8/50/71e60c5e9148c20de37c37f3e4cd1da1f63f7d0f7ea4c7e9c8a2f5c8d9e1/Django-4.2.7.tar.gz",
|
||||
wantName: "django",
|
||||
wantVersion: "4.2.7",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "sdist",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "real pandas wheel",
|
||||
urlPath: "/packages/a1/b2/c3d4e5f6/pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
|
||||
wantName: "pandas",
|
||||
wantVersion: "2.1.3",
|
||||
wantIsDownload: true,
|
||||
wantFileType: "wheel",
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
// Error cases
|
||||
{
|
||||
name: "empty URL path",
|
||||
urlPath: "",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantFileType: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "just slash",
|
||||
urlPath: "/",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantFileType: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid path without packages prefix",
|
||||
urlPath: "/files/ab/cd/requests-2.28.0.tar.gz",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantFileType: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unsupported file type",
|
||||
urlPath: "/packages/ab/cd/ef/readme.txt",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantFileType: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser := pypiFilesParser{}
|
||||
got, err := parser.ParseURL(tt.urlPath)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantName, got.GetName())
|
||||
assert.Equal(t, tt.wantVersion, got.GetVersion())
|
||||
assert.Equal(t, tt.wantIsDownload, got.IsFileDownload())
|
||||
|
||||
// Check file type via type assertion - must succeed for pypi packages
|
||||
pypiInfo, ok := got.(*pypiPackageInfo)
|
||||
assert.True(t, ok, "expected *pypiPackageInfo type")
|
||||
if ok {
|
||||
assert.Equal(t, tt.wantFileType, pypiInfo.FileType())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPypiOrgParser_ParseURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
urlPath string
|
||||
wantName string
|
||||
wantVersion string
|
||||
wantIsDownload bool
|
||||
wantErr bool
|
||||
}{
|
||||
// Simple API
|
||||
{
|
||||
name: "simple api package index",
|
||||
urlPath: "/simple/requests/",
|
||||
wantName: "requests",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "simple api package without trailing slash",
|
||||
urlPath: "/simple/django",
|
||||
wantName: "django",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "simple api normalized name",
|
||||
urlPath: "/simple/flask-restful/",
|
||||
wantName: "flask-restful",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
// JSON API
|
||||
{
|
||||
name: "json api package metadata",
|
||||
urlPath: "/pypi/requests/json",
|
||||
wantName: "requests",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "json api version metadata",
|
||||
urlPath: "/pypi/requests/2.28.0/json",
|
||||
wantName: "requests",
|
||||
wantVersion: "2.28.0",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "json api with normalized name",
|
||||
urlPath: "/pypi/flask-restful/0.3.10/json",
|
||||
wantName: "flask-restful",
|
||||
wantVersion: "0.3.10",
|
||||
wantIsDownload: false,
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
// Error cases
|
||||
{
|
||||
name: "empty URL path",
|
||||
urlPath: "",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "just slash",
|
||||
urlPath: "/",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown path prefix",
|
||||
urlPath: "/unknown/requests/",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "simple api missing package name",
|
||||
urlPath: "/simple/",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "json api missing package name",
|
||||
urlPath: "/pypi/json",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantIsDownload: false,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parser := pypiOrgParser{}
|
||||
got, err := parser.ParseURL(tt.urlPath)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantName, got.GetName())
|
||||
assert.Equal(t, tt.wantVersion, got.GetVersion())
|
||||
assert.Equal(t, tt.wantIsDownload, got.IsFileDownload())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWheelFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
wantName string
|
||||
wantVersion string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "simple wheel",
|
||||
filename: "requests-2.28.0-py3-none-any.whl",
|
||||
wantName: "requests",
|
||||
wantVersion: "2.28.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with cpython tag",
|
||||
filename: "numpy-1.24.0-cp311-cp311-linux_x86_64.whl",
|
||||
wantName: "numpy",
|
||||
wantVersion: "1.24.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with build tag",
|
||||
filename: "package-1.0.0-1-py3-none-any.whl",
|
||||
wantName: "package",
|
||||
wantVersion: "1.0.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wheel with underscore name",
|
||||
filename: "my_package-1.0.0-py3-none-any.whl",
|
||||
wantName: "my-package",
|
||||
wantVersion: "1.0.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid wheel - too few parts",
|
||||
filename: "invalid-1.0.0.whl",
|
||||
wantName: "",
|
||||
wantVersion: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseWheelFilename(tt.filename)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantName, got.GetName())
|
||||
assert.Equal(t, tt.wantVersion, got.GetVersion())
|
||||
assert.True(t, got.IsFileDownload())
|
||||
assert.Equal(t, "wheel", got.FileType())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSdistFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
wantName string
|
||||
wantVersion string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "simple tar.gz",
|
||||
filename: "requests-2.28.0.tar.gz",
|
||||
wantName: "requests",
|
||||
wantVersion: "2.28.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "zip format",
|
||||
filename: "django-4.2.0.zip",
|
||||
wantName: "django",
|
||||
wantVersion: "4.2.0",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "hyphenated name",
|
||||
filename: "Flask-RESTful-0.3.10.tar.gz",
|
||||
wantName: "flask-restful",
|
||||
wantVersion: "0.3.10",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "prerelease version",
|
||||
filename: "package-1.0.0rc1.tar.gz",
|
||||
wantName: "package",
|
||||
wantVersion: "1.0.0rc1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "dev version",
|
||||
filename: "package-0.1.0.dev1.tar.gz",
|
||||
wantName: "package",
|
||||
wantVersion: "0.1.0.dev1",
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseSdistFilename(tt.filename)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantName, got.GetName())
|
||||
assert.Equal(t, tt.wantVersion, got.GetVersion())
|
||||
assert.True(t, got.IsFileDownload())
|
||||
assert.Equal(t, "sdist", got.FileType())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDenormalizePyPIPackageName(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"requests", "requests"},
|
||||
{"Flask_RESTful", "flask-restful"},
|
||||
{"My_Package", "my-package"},
|
||||
{"UPPERCASE", "uppercase"},
|
||||
{"under_score", "under-score"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
result := denormalizePyPIPackageName(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPypiRegistryConfigForHostname(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hostname string
|
||||
expectConfig bool
|
||||
expectHost string
|
||||
}{
|
||||
{
|
||||
name: "exact match files.pythonhosted.org",
|
||||
hostname: "files.pythonhosted.org",
|
||||
expectConfig: true,
|
||||
expectHost: "files.pythonhosted.org",
|
||||
},
|
||||
{
|
||||
name: "exact match pypi.org",
|
||||
hostname: "pypi.org",
|
||||
expectConfig: true,
|
||||
expectHost: "pypi.org",
|
||||
},
|
||||
{
|
||||
name: "subdomain match",
|
||||
hostname: "cdn.files.pythonhosted.org",
|
||||
expectConfig: true,
|
||||
expectHost: "files.pythonhosted.org",
|
||||
},
|
||||
{
|
||||
name: "test pypi",
|
||||
hostname: "test.pypi.org",
|
||||
expectConfig: true,
|
||||
expectHost: "test.pypi.org",
|
||||
},
|
||||
{
|
||||
name: "unknown hostname",
|
||||
hostname: "example.com",
|
||||
expectConfig: false,
|
||||
expectHost: "",
|
||||
},
|
||||
{
|
||||
name: "partial match should not work",
|
||||
hostname: "fakepypi.org",
|
||||
expectConfig: false,
|
||||
expectHost: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
config := pypiRegistryDomains.GetConfigForHostname(tt.hostname)
|
||||
|
||||
if !tt.expectConfig {
|
||||
assert.Nil(t, config)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NotNil(t, config)
|
||||
assert.Equal(t, tt.expectHost, config.Host)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPypiRegistryDomains_ContainsHostname(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hostname string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "exact match",
|
||||
hostname: "files.pythonhosted.org",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "subdomain match",
|
||||
hostname: "cdn.files.pythonhosted.org",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
hostname: "example.com",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := pypiRegistryDomains.ContainsHostname(tt.hostname)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user