diff --git a/cmd/golang/go.go b/cmd/golang/go.go new file mode 100644 index 0000000..52582f5 --- /dev/null +++ b/cmd/golang/go.go @@ -0,0 +1,108 @@ +// Package golang implements the experimental `pmg go` command. The package is +// named golang (not go) to avoid shadowing the toolchain name as a package +// path; the user-facing command is still `pmg go`. +package golang + +import ( + "context" + "fmt" + + "github.com/safedep/dry/usefulerror" + "github.com/safedep/pmg/config" + "github.com/safedep/pmg/errcodes" + "github.com/safedep/pmg/internal/analytics" + "github.com/safedep/pmg/internal/flows" + "github.com/safedep/pmg/internal/ui" + "github.com/safedep/pmg/packagemanager" + "github.com/safedep/pmg/proxy/certmanager" + "github.com/safedep/pmg/truststore" + "github.com/spf13/cobra" +) + +func NewGoCommand() *cobra.Command { + return &cobra.Command{ + Use: "go [action] [module]", + Short: "Guard go module downloads (experimental)", + DisableFlagParsing: true, + RunE: func(cmd *cobra.Command, args []string) error { + err := executeGoFlow(cmd.Context(), args) + if err != nil { + ui.ExitFromCommandError(err) + } + + return nil + }, + } +} + +func executeGoFlow(ctx context.Context, args []string) error { + analytics.TrackCommandGo() + + packageManager, err := packagemanager.NewGoPackageManager(packagemanager.DefaultGoPackageManagerConfig()) + if err != nil { + return fmt.Errorf("failed to create go package manager: %w", err) + } + + parsedCommand, err := packageManager.ParseCommand(args) + if err != nil { + return fmt.Errorf("failed to parse command: %w", err) + } + + if !config.Get().IsProxyModeEnabled() { + return errGoRequiresProxyMode() + } + + if err := requireTrustedCA(); err != nil { + return err + } + + return flows.ProxyFlow(packageManager, packagemanager.NewNoopPackageResolver()).Run(ctx, args, parsedCommand) +} + +// requireTrustedCA fails fast when Go cannot trust PMG's MITM CA. Go's +// crypto/x509 ignores SSL_CERT_FILE on macOS and Windows and verifies TLS +// against the OS trust store only, so without an OS-trusted persisted CA every +// module download would fail mid-build with an opaque x509 error. Linux honors +// the injected SSL_CERT_FILE bundle, so any CA (persisted or ephemeral) works. +func requireTrustedCA() error { + if !truststore.UserScopeSupported() { + return nil + } + + if _, err := certmanager.LoadCA(config.Get().ConfigDir()); err != nil { + return errGoCertNotTrusted(err) + } + + user, system, err := truststore.Status(certmanager.CACommonName) + if err != nil { + return errGoCertNotTrusted(err) + } + + if !user && !system { + return errGoCertNotTrusted(nil) + } + + return nil +} + +func errGoRequiresProxyMode() error { + return usefulerror.NewUsefulError(). + WithCode(errcodes.InvalidArgument). + WithHumanError("Go support requires proxy mode, which is disabled in your configuration."). + WithHelp("Enable proxy mode (proxy.enabled: true in the PMG config) and retry."). + WithMsg("go requires proxy mode") +} + +func errGoCertNotTrusted(cause error) error { + err := usefulerror.NewUsefulError(). + WithCode(errcodes.CertTrustStore). + WithHumanError("Go ignores PMG's injected CA bundle on this OS; the PMG proxy CA must be trusted in the OS trust store."). + WithHelp("Run `pmg setup cert install` to install and trust the PMG proxy CA, then retry."). + WithMsg("pmg proxy CA is not trusted in the OS trust store") + + if cause != nil { + return err.Wrap(cause) + } + + return err +} diff --git a/docs/proxy-mode.md b/docs/proxy-mode.md index c7526b7..228857c 100644 --- a/docs/proxy-mode.md +++ b/docs/proxy-mode.md @@ -69,6 +69,37 @@ Legacy variables `PMG_PROXY_MODE` and `PMG_PROXY_INSTALL_ONLY` (for the old flat | `uv` | โœ… | | `uvx` | โœ… | | `poetry` | โœ… | +| `go` | ๐Ÿงช experimental | + +### Go (experimental) + +`pmg go` guards Go module downloads through the same proxy flow. It is +experimental and opt-in: it only runs when invoked explicitly as `pmg go ...` +and is deliberately excluded from `pmg setup` shell aliases and PATH shims. + +
+How it differs from npm/PyPI + +- The module proxy host comes from the effective `GOPROXY` (including + `go env -w` values), not a fixed registry. PMG intercepts whatever HTTPS + proxies are configured and rewrites the child's `GOPROXY` to a fail-closed, + comma-joined list: `direct` is removed (a module PMG cannot inspect fails + instead of silently bypassing analysis) and pipe separators collapse to + comma so a block is terminal. +- Malware analysis and dependency cooldown run on the `.zip` source download โ€” + the only GOPROXY endpoint that delivers code. `.info`/`.mod` metadata passes + through (cooldown reads the publish time from `.info` without modifying it). +- `sum.golang.org` is never MITM'd and `/sumdb/` requests pass through + unmodified, so Go's checksum-database verification stays fully intact. + Toolchain downloads (`golang.org/toolchain`) are allowed on Go's own + checksum verification. +- On macOS and Windows, Go only trusts the OS trust store, so + `pmg setup cert install` is required first; `pmg go` fails fast with + instructions if the PMG CA is not trusted. Linux works out of the box. +- Modules matching `GOPRIVATE`/`GONOPROXY` are fetched directly from their + VCS host and are not analyzed; PMG warns when these are set. + +
## References diff --git a/go.mod b/go.mod index fa912f2..1641b47 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 + golang.org/x/mod v0.33.0 golang.org/x/net v0.51.0 golang.org/x/sync v0.20.0 golang.org/x/sys v0.43.0 @@ -83,7 +84,6 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/mod v0.33.0 // indirect golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/internal/analytics/event.go b/internal/analytics/event.go index 35883ea..c4f180c 100644 --- a/internal/analytics/event.go +++ b/internal/analytics/event.go @@ -12,6 +12,7 @@ const ( eventCommandPoetry = "pmg_command_poetry" eventCommandPipx = "pmg_command_pipx" eventCommandUvx = "pmg_command_uvx" + eventCommandGo = "pmg_command_go" eventCommandNpx = "pmg_command_npx" eventCommandPnpx = "pmg_command_pnpx" @@ -73,6 +74,10 @@ func TrackCommandUvx() { TrackEvent(eventCommandUvx) } +func TrackCommandGo() { + TrackEvent(eventCommandGo) +} + func TrackCommandGenerateEnvDocker() { TrackEvent(eventPmgGenerateEnvDocker) } diff --git a/internal/flows/proxy_flow.go b/internal/flows/proxy_flow.go index 40ee591..433f30c 100644 --- a/internal/flows/proxy_flow.go +++ b/internal/flows/proxy_flow.go @@ -173,9 +173,20 @@ func (f *proxyFlow) Run(ctx context.Context, args []string, parsedCmd *packagema } } + // Package managers with run-specific proxy routing (Go's user-configurable + // GOPROXY) contribute extra child env vars and dynamic MITM hosts. + routing := &packagemanager.ProxyRouting{} + if provider, ok := f.pm.(packagemanager.ProxyRoutingProvider); ok { + routing, err = provider.ProxyRouting(ctx) + if err != nil { + return fmt.Errorf("failed to resolve proxy routing for %s: %w", f.pm.Name(), err) + } + } + // Create ecosystem-specific interceptor using factory factory := interceptors.NewInterceptorFactory(malysisAnalyzer, cache, statsCollector, confirmationChan, interceptors.InterceptorContext{ - PinnedVersions: pinnedVersions, + PinnedVersions: pinnedVersions, + GoProxyBaseURLs: routing.MITMHosts, }) interceptor, err := factory.CreateInterceptor(ecosystem) if err != nil { @@ -212,7 +223,7 @@ func (f *proxyFlow) Run(ctx context.Context, args []string, parsedCmd *packagema PackageManagerName: f.pm.Name(), DryRun: cfg.DryRun, Mode: runner.ExecutionModeAuto, - EnvOverrides: packagemanager.EnvVarForProxy(proxyAddr, caCertPath), + EnvOverrides: append(packagemanager.EnvVarForProxy(proxyAddr, caCertPath), routing.ExtraEnv...), DirectEnvOverrides: ciEnvOverride(), BeforeDirectRun: func() error { log.Debugf("Executing proxy for non interactive TTY") diff --git a/main.go b/main.go index 8550a12..3d42efb 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "github.com/safedep/pmg/cmd/cloud" configCmd "github.com/safedep/pmg/cmd/config" "github.com/safedep/pmg/cmd/executors" + golangCmd "github.com/safedep/pmg/cmd/golang" landlockCmd "github.com/safedep/pmg/cmd/landlock" "github.com/safedep/pmg/cmd/npm" proxyCmd "github.com/safedep/pmg/cmd/proxy" @@ -161,6 +162,7 @@ func main() { cmd.AddCommand(pypi.NewPoetryCommand()) cmd.AddCommand(executors.NewPipxCommand()) cmd.AddCommand(executors.NewUvxCommand()) + cmd.AddCommand(golangCmd.NewGoCommand()) cmd.AddCommand(proxyCmd.NewProxyCommand()) cmd.AddCommand(version.NewVersionCommand()) cmd.AddCommand(setup.NewSetupCommand()) diff --git a/packagemanager/golang.go b/packagemanager/golang.go new file mode 100644 index 0000000..bbb3293 --- /dev/null +++ b/packagemanager/golang.go @@ -0,0 +1,158 @@ +package packagemanager + +import ( + "slices" + "strings" + + packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" + "golang.org/x/mod/semver" +) + +type GoPackageManagerConfig struct { + CommandName string + + // InstallCommands accept module@version args on the command line, used to + // extract pinned versions for cooldown reporting. + InstallCommands []string + + // NonDownloadCommands never load packages and therefore never fetch + // modules. Deliberately minimal: fmt/clean/vet/fix/doc/mod-graph load + // packages and can download already-required modules on a cold cache even + // under -mod=readonly, so they are excluded on purpose and run with the + // proxy. + NonDownloadCommands []string +} + +func DefaultGoPackageManagerConfig() GoPackageManagerConfig { + return GoPackageManagerConfig{ + CommandName: "go", + InstallCommands: []string{"get", "install", "run"}, + NonDownloadCommands: []string{"version", "env", "help"}, + } +} + +type goPackageManager struct { + Config GoPackageManagerConfig +} + +func NewGoPackageManager(config GoPackageManagerConfig) (*goPackageManager, error) { + return &goPackageManager{Config: config}, nil +} + +var _ PackageManager = &goPackageManager{} + +func (g *goPackageManager) Name() string { + return g.Config.CommandName +} + +func (g *goPackageManager) Ecosystem() packagev1.Ecosystem { + return packagev1.Ecosystem_ECOSYSTEM_GO +} + +func (g *goPackageManager) ParseCommand(args []string) (*ParsedCommand, error) { + if len(args) > 0 && args[0] == g.Config.CommandName { + args = args[1:] + } + + parsed := &ParsedCommand{Command: Command{Exe: g.Config.CommandName, Args: args}} + + subcmd, rest := goFirstNonFlagArg(args) + if subcmd == "" { + return parsed, nil + } + + if slices.Contains(g.Config.NonDownloadCommands, subcmd) { + parsed.IsKnownNonDownloadCommand = true + return parsed, nil + } + + if slices.Contains(g.Config.InstallCommands, subcmd) { + parsed.InstallTargets = goRemoteModuleTargets(rest) + return parsed, nil + } + + if subcmd == "mod" { + modCmd, modRest := goFirstNonFlagArg(rest) + switch modCmd { + case "tidy": + parsed.IsManifestInstall = true + parsed.ManifestFiles = []string{"go.mod"} + case "download": + parsed.InstallTargets = goRemoteModuleTargets(modRest) + if len(parsed.InstallTargets) == 0 { + parsed.IsManifestInstall = true + parsed.ManifestFiles = []string{"go.mod"} + } + } + } + + return parsed, nil +} + +func goFirstNonFlagArg(args []string) (string, []string) { + for i, arg := range args { + if strings.HasPrefix(arg, "-") { + continue + } + return arg, args[i+1:] + } + return "", nil +} + +// goRemoteModuleTargets extracts remote module targets (module[@version]) from +// command args, skipping flags, local paths and meta-patterns. Version queries +// (@latest, branch names, commit hashes) are passed through for go to resolve; +// only canonical semver counts as an explicit version for cooldown reporting. +func goRemoteModuleTargets(args []string) []*PackageInstallTarget { + var targets []*PackageInstallTarget + + for _, arg := range args { + if strings.HasPrefix(arg, "-") || !isGoRemoteModuleTarget(arg) { + continue + } + + name, version := arg, "" + if at := strings.LastIndex(arg, "@"); at > 0 { + name, version = arg[:at], arg[at+1:] + } + + targets = append(targets, &PackageInstallTarget{ + PackageVersion: &packagev1.PackageVersion{ + Package: &packagev1.Package{ + Ecosystem: packagev1.Ecosystem_ECOSYSTEM_GO, + Name: name, + }, + Version: version, + }, + IsExplicitVersion: semver.IsValid(version) && semver.Canonical(version) == version, + }) + } + + return targets +} + +// isGoRemoteModuleTarget discriminates a remote module target from a local +// path or meta-pattern: a remote target's first path segment is a domain +// (contains a dot), so `go install ./cmd/foo` and `go build ./...` yield no +// targets while `go get github.com/x/y@v1.2.3` does. +func isGoRemoteModuleTarget(target string) bool { + target = strings.TrimSpace(target) + if target == "" || target == "." || target == ".." { + return false + } + + if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || strings.HasPrefix(target, "/") { + return false + } + + if strings.Contains(target, `\`) { + return false + } + + firstSegment, _, _ := strings.Cut(target, "/") + if firstSegment == "..." { + return false + } + + return strings.Contains(firstSegment, ".") +} diff --git a/packagemanager/golang_test.go b/packagemanager/golang_test.go new file mode 100644 index 0000000..adfabda --- /dev/null +++ b/packagemanager/golang_test.go @@ -0,0 +1,164 @@ +package packagemanager + +import ( + "testing" + + packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoPackageManagerParseCommand(t *testing.T) { + type target struct { + name string + version string + explicit bool + } + + cases := []struct { + name string + args []string + nonDownload bool + manifestInstall bool + targets []target + wantManifestFiles []string + }{ + { + name: "go version is non-download", + args: []string{"go", "version"}, + nonDownload: true, + }, + { + name: "go env is non-download", + args: []string{"go", "env", "GOPROXY"}, + nonDownload: true, + }, + { + name: "go vet is not non-download (can fetch on cold cache)", + args: []string{"go", "vet", "./..."}, + }, + { + name: "go fmt is not non-download (runs go list, can fetch on cold cache)", + args: []string{"go", "fmt", "./..."}, + }, + { + name: "go clean is not non-download (loads packages, can fetch on cold cache)", + args: []string{"go", "clean", "./..."}, + }, + { + name: "go build runs with proxy", + args: []string{"go", "build", "./..."}, + }, + { + name: "go get with canonical version", + args: []string{"go", "get", "github.com/x/y@v1.2.3"}, + targets: []target{{name: "github.com/x/y", version: "v1.2.3", explicit: true}}, + }, + { + name: "go get with pseudo-version is explicit", + args: []string{"go", "get", "github.com/x/y@v0.0.0-20191109021931-daa7c04131f5"}, + targets: []target{{name: "github.com/x/y", version: "v0.0.0-20191109021931-daa7c04131f5", explicit: true}}, + }, + { + name: "go install with latest query is not explicit", + args: []string{"go", "install", "github.com/x/y/cmd/y@latest"}, + targets: []target{{name: "github.com/x/y/cmd/y", version: "latest", explicit: false}}, + }, + { + name: "go get with branch query is not explicit", + args: []string{"go", "get", "github.com/x/y@master"}, + targets: []target{{name: "github.com/x/y", version: "master", explicit: false}}, + }, + { + name: "go install local path yields no target", + args: []string{"go", "install", "./cmd/foo"}, + }, + { + name: "go run current dir yields no target", + args: []string{"go", "run", "."}, + }, + { + name: "go get without version", + args: []string{"go", "get", "example.com/m"}, + targets: []target{{name: "example.com/m", version: "", explicit: false}}, + }, + { + name: "flags before target are skipped", + args: []string{"go", "get", "-u", "example.com/m@v2.0.0"}, + targets: []target{{name: "example.com/m", version: "v2.0.0", explicit: true}}, + }, + { + name: "go mod tidy is manifest install", + args: []string{"go", "mod", "tidy"}, + manifestInstall: true, + wantManifestFiles: []string{"go.mod"}, + }, + { + name: "go mod download without args is manifest install", + args: []string{"go", "mod", "download"}, + manifestInstall: true, + wantManifestFiles: []string{"go.mod"}, + }, + { + name: "go mod download with module", + args: []string{"go", "mod", "download", "example.com/m@v1.0.0"}, + targets: []target{{name: "example.com/m", version: "v1.0.0", explicit: true}}, + }, + { + name: "no subcommand", + args: []string{"go"}, + }, + } + + pm, err := NewGoPackageManager(DefaultGoPackageManagerConfig()) + require.NoError(t, err) + + assert.Equal(t, "go", pm.Name()) + assert.Equal(t, packagev1.Ecosystem_ECOSYSTEM_GO, pm.Ecosystem()) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parsed, err := pm.ParseCommand(tc.args) + require.NoError(t, err) + + assert.Equal(t, tc.nonDownload, parsed.IsKnownNonDownloadCommand) + assert.Equal(t, tc.manifestInstall, parsed.IsManifestInstall) + assert.Equal(t, tc.wantManifestFiles, parsed.ManifestFiles) + + require.Len(t, parsed.InstallTargets, len(tc.targets)) + for i, want := range tc.targets { + got := parsed.InstallTargets[i] + assert.Equal(t, want.name, got.PackageVersion.GetPackage().GetName()) + assert.Equal(t, want.version, got.PackageVersion.GetVersion()) + assert.Equal(t, want.explicit, got.IsExplicitVersion) + assert.Equal(t, packagev1.Ecosystem_ECOSYSTEM_GO, got.PackageVersion.GetPackage().GetEcosystem()) + } + }) + } +} + +func TestIsGoRemoteModuleTarget(t *testing.T) { + cases := []struct { + target string + want bool + }{ + {"github.com/x/y", true}, + {"example.com/m@v1.0.0", true}, + {"gopkg.in/yaml.v3", true}, + {".", false}, + {"..", false}, + {"./cmd/foo", false}, + {"../pkg", false}, + {"/abs/path", false}, + {"./...", false}, + {"...", false}, + {`a\b`, false}, + {"", false}, + {"fmt", false}, + {"cmd/foo", false}, + } + + for _, tc := range cases { + assert.Equal(t, tc.want, isGoRemoteModuleTarget(tc.target), "target %q", tc.target) + } +} diff --git a/packagemanager/goproxy.go b/packagemanager/goproxy.go new file mode 100644 index 0000000..b24003b --- /dev/null +++ b/packagemanager/goproxy.go @@ -0,0 +1,142 @@ +package packagemanager + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os/exec" + "strings" + + "github.com/safedep/dry/log" +) + +// ProxyRouting is per-run routing a package manager contributes to the proxy +// flow before the proxied child process launches. ExtraEnv is appended to the +// standard proxy env injection. MITMHosts maps registry hostnames the proxy +// must intercept dynamically to the upstream base URL packages are served +// under (scheme + host + optional path prefix) โ€” Go's module proxy is +// user-configurable via GOPROXY, unlike npm/PyPI's fixed registry hosts. +type ProxyRouting struct { + ExtraEnv []string + MITMHosts map[string]string +} + +// ProxyRoutingProvider is implemented by package managers that need +// run-specific proxy routing. +type ProxyRoutingProvider interface { + ProxyRouting(ctx context.Context) (*ProxyRouting, error) +} + +const defaultGoProxyURL = "https://proxy.golang.org" + +var _ ProxyRoutingProvider = &goPackageManager{} + +// ProxyRouting computes the child's GOPROXY and the module-proxy hosts to +// MITM from the effective go env. GOPRIVATE/GONOPROXY are left untouched for +// the user's private modules but surfaced as a warning since matching modules +// bypass analysis; GOINSECURE is cleared so module traffic cannot be +// downgraded to plaintext HTTP. +func (g *goPackageManager) ProxyRouting(ctx context.Context) (*ProxyRouting, error) { + env, err := readEffectiveGoEnv(ctx, "GOPROXY", "GOPRIVATE", "GONOPROXY", "GOINSECURE") + if err != nil { + return nil, err + } + + childGoProxy, mitmHosts := normalizeGoProxy(env["GOPROXY"]) + + for _, key := range []string{"GOPRIVATE", "GONOPROXY"} { + if v := env[key]; v != "" { + log.Warnf("%s=%q: matching modules are fetched directly from their VCS host and are NOT analyzed by PMG", key, v) + } + } + + routing := &ProxyRouting{ + ExtraEnv: []string{fmt.Sprintf("GOPROXY=%s", childGoProxy)}, + MITMHosts: mitmHosts, + } + + if env["GOINSECURE"] != "" { + log.Warnf("GOINSECURE is set; PMG clears it for this run so module traffic cannot be downgraded to plaintext HTTP") + routing.ExtraEnv = append(routing.ExtraEnv, "GOINSECURE=") + } + + return routing, nil +} + +// readEffectiveGoEnv reads go env values honoring both the process environment +// and the user's persisted GOENV file (go env -w), which plain os.Getenv would +// miss. +func readEffectiveGoEnv(ctx context.Context, keys ...string) (map[string]string, error) { + out, err := exec.CommandContext(ctx, "go", append([]string{"env", "-json"}, keys...)...).Output() + if err != nil { + return nil, fmt.Errorf("failed to read go env (is the Go toolchain installed and on PATH?): %w", err) + } + + values := map[string]string{} + if err := json.Unmarshal(out, &values); err != nil { + return nil, fmt.Errorf("failed to parse go env output: %w", err) + } + + return values, nil +} + +// normalizeGoProxy rebuilds the child's GOPROXY as a fail-closed proxy list +// and returns the hostnames to MITM mapped to their upstream base URL (used +// for base-path stripping and out-of-band .info fetches): +// +// - `direct` entries are dropped so a module PMG cannot inspect fails with +// an error instead of silently bypassing analysis via a VCS fetch. +// - Pipe (|) separators collapse to comma so a PMG block (HTTP 403) is +// terminal rather than falling through to the next entry. +// - `off` is kept: it is already fail-closed (no network at all). +// - Unschemed entries (go treats them as https) are rewritten with an +// explicit https:// scheme so they are unambiguous and interceptable. +// - file:// proxies are local (no network) and kept verbatim; there is no +// host to intercept. +// +// If nothing remains (GOPROXY was direct-only), the public Go proxy is +// injected so module downloads stay analyzable. +func normalizeGoProxy(goproxy string) (child string, mitmHosts map[string]string) { + if strings.TrimSpace(goproxy) == "" { + goproxy = defaultGoProxyURL + ",direct" + } + + mitmHosts = map[string]string{} + + var kept []string + droppedDirect := false + for _, entry := range strings.FieldsFunc(goproxy, func(r rune) bool { return r == ',' || r == '|' }) { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + + if entry == "direct" { + droppedDirect = true + continue + } + + if entry != "off" && !strings.Contains(entry, "://") { + entry = "https://" + entry + } + + kept = append(kept, entry) + + if u, err := url.Parse(entry); err == nil && u.Hostname() != "" && (u.Scheme == "https" || u.Scheme == "http") { + mitmHosts[u.Hostname()] = strings.TrimSuffix(entry, "/") + } + } + + if droppedDirect { + log.Warnf("Removed 'direct' from GOPROXY for this run: modules unavailable on the module proxy fail instead of bypassing analysis") + } + + if len(kept) == 0 { + log.Warnf("GOPROXY=%q has no usable module proxy; PMG routes module downloads via %s for analysis", goproxy, defaultGoProxyURL) + kept = append(kept, defaultGoProxyURL) + mitmHosts["proxy.golang.org"] = defaultGoProxyURL + } + + return strings.Join(kept, ","), mitmHosts +} diff --git a/packagemanager/goproxy_test.go b/packagemanager/goproxy_test.go new file mode 100644 index 0000000..bb18e1e --- /dev/null +++ b/packagemanager/goproxy_test.go @@ -0,0 +1,94 @@ +package packagemanager + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizeGoProxy(t *testing.T) { + cases := []struct { + name string + goproxy string + wantChild string + wantHosts map[string]string + }{ + { + name: "empty defaults to public proxy without direct", + goproxy: "", + wantChild: "https://proxy.golang.org", + wantHosts: map[string]string{"proxy.golang.org": "https://proxy.golang.org"}, + }, + { + name: "default value drops direct", + goproxy: "https://proxy.golang.org,direct", + wantChild: "https://proxy.golang.org", + wantHosts: map[string]string{"proxy.golang.org": "https://proxy.golang.org"}, + }, + { + name: "pipe separator collapses to comma", + goproxy: "https://corp.example.com|https://proxy.golang.org|direct", + wantChild: "https://corp.example.com,https://proxy.golang.org", + wantHosts: map[string]string{ + "corp.example.com": "https://corp.example.com", + "proxy.golang.org": "https://proxy.golang.org", + }, + }, + { + name: "direct only injects public proxy", + goproxy: "direct", + wantChild: "https://proxy.golang.org", + wantHosts: map[string]string{"proxy.golang.org": "https://proxy.golang.org"}, + }, + { + name: "off is kept and nothing is intercepted", + goproxy: "off", + wantChild: "off", + wantHosts: map[string]string{}, + }, + { + name: "proxy with base path keeps full base URL", + goproxy: "https://corp.example.com:8443/goproxy,direct", + wantChild: "https://corp.example.com:8443/goproxy", + wantHosts: map[string]string{"corp.example.com": "https://corp.example.com:8443/goproxy"}, + }, + { + name: "unschemed entry defaults to https (go behavior)", + goproxy: "proxy.corp.internal,direct", + wantChild: "https://proxy.corp.internal", + wantHosts: map[string]string{"proxy.corp.internal": "https://proxy.corp.internal"}, + }, + { + name: "unschemed entry with port", + goproxy: "proxy.corp.internal:8443", + wantChild: "https://proxy.corp.internal:8443", + wantHosts: map[string]string{"proxy.corp.internal": "https://proxy.corp.internal:8443"}, + }, + { + name: "http proxy is kept and intercepted", + goproxy: "http://insecure.example.com", + wantChild: "http://insecure.example.com", + wantHosts: map[string]string{"insecure.example.com": "http://insecure.example.com"}, + }, + { + name: "file proxy kept verbatim with no host", + goproxy: "file:///var/goproxy,direct", + wantChild: "file:///var/goproxy", + wantHosts: map[string]string{}, + }, + { + name: "whitespace and empty entries are ignored", + goproxy: " https://proxy.golang.org , ,direct ", + wantChild: "https://proxy.golang.org", + wantHosts: map[string]string{"proxy.golang.org": "https://proxy.golang.org"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + child, hosts := normalizeGoProxy(tc.goproxy) + assert.Equal(t, tc.wantChild, child) + assert.Equal(t, tc.wantHosts, hosts) + }) + } +} diff --git a/packagemanager/noop_resolver.go b/packagemanager/noop_resolver.go new file mode 100644 index 0000000..f78aa1b --- /dev/null +++ b/packagemanager/noop_resolver.go @@ -0,0 +1,25 @@ +package packagemanager + +import ( + "context" + "fmt" + + packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" +) + +// noopPackageResolver satisfies PackageResolver for flows that never resolve +// dependencies up front, such as the proxy flow where every download is +// intercepted and analyzed on the wire. +type noopPackageResolver struct{} + +func NewNoopPackageResolver() PackageResolver { + return noopPackageResolver{} +} + +func (noopPackageResolver) ResolveLatestVersion(context.Context, *packagev1.Package) (*packagev1.PackageVersion, error) { + return nil, fmt.Errorf("package resolution is not supported by the noop resolver") +} + +func (noopPackageResolver) ResolveDependencies(context.Context, *packagev1.PackageVersion) ([]*packagev1.PackageVersion, error) { + return nil, fmt.Errorf("dependency resolution is not supported by the noop resolver") +} diff --git a/proxy/interceptors/audit_logger.go b/proxy/interceptors/audit_logger.go index 3474570..d3ab8ab 100644 --- a/proxy/interceptors/audit_logger.go +++ b/proxy/interceptors/audit_logger.go @@ -46,6 +46,16 @@ func (i *AuditLoggerInterceptor) HandleRequest(ctx *proxy.RequestContext) (*prox return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil } +// wellKnownGoHosts are the default Go module-proxy and checksum-database +// hosts. Custom GOPROXY hosts are dynamic (known only to the Go interceptor's +// per-run config) and intentionally still surface here as observed hosts. +var wellKnownGoHosts = map[string]bool{ + "proxy.golang.org": true, + "sum.golang.org": true, +} + func (i *AuditLoggerInterceptor) isKnownRegistryHost(hostname string) bool { - return npmRegistryDomains.ContainsHostname(hostname) || pypiRegistryDomains.ContainsHostname(hostname) + return npmRegistryDomains.ContainsHostname(hostname) || + pypiRegistryDomains.ContainsHostname(hostname) || + wellKnownGoHosts[hostname] } diff --git a/proxy/interceptors/factory.go b/proxy/interceptors/factory.go index db6ad97..beb2cbd 100644 --- a/proxy/interceptors/factory.go +++ b/proxy/interceptors/factory.go @@ -13,6 +13,13 @@ import ( // (analyzer, cache, stats), this holds context specific to the current run. type InterceptorContext struct { PinnedVersions map[string]string + + // GoProxyBaseURLs maps module-proxy hostnames from the user's effective + // GOPROXY to their upstream base URL (scheme + host + optional path + // prefix). The Go interceptor MITMs and analyzes these hosts; Go is the + // only ecosystem whose registry hosts are user-configurable rather than + // fixed. + GoProxyBaseURLs map[string]string } // InterceptorFactory creates ecosystem-specific interceptors for the proxy @@ -63,6 +70,15 @@ func (f *InterceptorFactory) CreateInterceptor(ecosystem packagev1.Ecosystem) (p f.execContext, ), nil + case packagev1.Ecosystem_ECOSYSTEM_GO: + return NewGoRegistryInterceptor( + f.analyzer, + f.cache, + f.statsCollector, + f.confirmationChan, + f.execContext, + ), nil + default: return nil, fmt.Errorf("proxy-based interception not yet supported for ecosystem: %s", ecosystem.String()) } @@ -73,6 +89,7 @@ func SupportedEcosystems() []packagev1.Ecosystem { return []packagev1.Ecosystem{ packagev1.Ecosystem_ECOSYSTEM_NPM, packagev1.Ecosystem_ECOSYSTEM_PYPI, + packagev1.Ecosystem_ECOSYSTEM_GO, } } diff --git a/proxy/interceptors/go_cooldown.go b/proxy/interceptors/go_cooldown.go new file mode 100644 index 0000000..b0ea6c9 --- /dev/null +++ b/proxy/interceptors/go_cooldown.go @@ -0,0 +1,189 @@ +package interceptors + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" + "github.com/safedep/dry/log" + pmgconfig "github.com/safedep/pmg/config" + "github.com/safedep/pmg/internal/audit" + "github.com/safedep/pmg/proxy" + gomodule "golang.org/x/mod/module" +) + +// goCooldownHandler enforces dependency cooldown for Go modules. Unlike npm, +// there is no metadata to strip: the version go requests is already resolved +// by the time the proxy sees it. Instead the publish timestamp is captured +// from the .info response go fetches before each .zip, and an in-window .zip +// download is blocked with HTTP 403 โ€” terminal, because the child GOPROXY is +// normalized to a comma-joined list with no direct fallback. +type goCooldownHandler struct { + statsCollector *AnalysisStatsCollector + + mu sync.Mutex + publishTimes map[string]time.Time +} + +func newGoCooldownHandler(statsCollector *AnalysisStatsCollector) *goCooldownHandler { + return &goCooldownHandler{ + statsCollector: statsCollector, + publishTimes: map[string]time.Time{}, + } +} + +func goModuleVersionKey(module, version string) string { + return module + "@" + version +} + +// HandleInfoRequest reads the .info response body without altering it and +// caches the version's publish time for the upcoming .zip request. +func (h *goCooldownHandler) HandleInfoRequest(ctx *proxy.RequestContext, module, version string) (*proxy.InterceptorResponse, error) { + // Force an uncompressed, non-conditional response so the body is parseable + // JSON rather than raw gzip bytes or an empty 304 (same as the npm + // metadata modifier). + ctx.Headers.Set("Accept-Encoding", "identity") + ctx.Headers.Del("If-None-Match") + ctx.Headers.Del("If-Modified-Since") + + modifier := func(statusCode int, headers http.Header, body []byte) (int, http.Header, []byte, error) { + if statusCode != http.StatusOK { + return statusCode, headers, body, nil + } + + var info struct { + Time time.Time `json:"Time"` + } + if err := json.Unmarshal(body, &info); err != nil || info.Time.IsZero() { + log.Warnf("[%s] Cooldown: failed to parse publish time from .info for %s@%s", ctx.RequestID, module, version) + return statusCode, headers, body, nil + } + + h.mu.Lock() + h.publishTimes[goModuleVersionKey(module, version)] = info.Time + h.mu.Unlock() + + return statusCode, headers, body, nil + } + + return &proxy.InterceptorResponse{ + Action: proxy.ActionModifyResponse, + ResponseModifier: modifier, + }, nil +} + +// CheckZipDownload blocks the module zip when its publish time is within the +// cooldown window. handled=false lets the request continue to malware +// analysis. When the publish time was not observed on the wire (go served +// .info from its local module cache, common on machines that used go before +// PMG), it is fetched out-of-band from the upstream proxy; only if that also +// fails does cooldown fail open โ€” malware analysis still runs. +func (h *goCooldownHandler) CheckZipDownload(ctx *proxy.RequestContext, baseURL, module, version string, cooldownDays int) (*proxy.InterceptorResponse, bool) { + skip := pmgconfig.CooldownSkip(packagev1.Ecosystem_ECOSYSTEM_GO, module) + if skip.SkipAll || pmgconfig.IsTrustedPackageRef(packagev1.Ecosystem_ECOSYSTEM_GO, module, version) { + return nil, false + } + + h.mu.Lock() + publishTime, ok := h.publishTimes[goModuleVersionKey(module, version)] + h.mu.Unlock() + + if !ok { + publishTime, ok = h.fetchPublishTime(ctx, baseURL, module, version) + } + + if !ok { + log.Warnf("[%s] Cooldown: no publish time available for %s@%s; cooldown not enforced for this download", ctx.RequestID, module, version) + return nil, false + } + + within, daysAgo, daysLeft := cooldownIsWithinWindow(publishTime, cooldownDays) + if !within { + return nil, false + } + + if skip.ExemptsVersion(version) { + auditCooldownSkips(ctx.RequestID, packagev1.Ecosystem_ECOSYSTEM_GO, module, cooldownExemptions{skipListed: []string{version}}) + return nil, false + } + + log.Infof("[%s] Cooldown: blocking %s@%s published %d day(s) ago (%d day cooldown, %d remaining)", + ctx.RequestID, module, version, daysAgo, cooldownDays, daysLeft) + + if h.statsCollector != nil { + h.statsCollector.RecordCooldownBlocked(module, version, publishTime, daysAgo, daysLeft, cooldownDays) + } + + pv := &packagev1.PackageVersion{} + pv.SetPackage(&packagev1.Package{}) + pv.GetPackage().SetName(module) + pv.GetPackage().SetEcosystem(packagev1.Ecosystem_ECOSYSTEM_GO) + pv.SetVersion(version) + audit.LogDependencyCooldown(pv, publishTime, cooldownDays, daysAgo, daysLeft) + + message := fmt.Sprintf("Package blocked by dependency cooldown: GO/%s@%s\n\nPublished %d day(s) ago; cooldown window is %d day(s) (%d remaining).", + module, version, daysAgo, cooldownDays, daysLeft) + + return &proxy.InterceptorResponse{ + Action: proxy.ActionBlock, + BlockCode: http.StatusForbidden, + BlockMessage: message, + }, true +} + +// goInfoFetchClient fetches .info out-of-band, straight to the upstream proxy +// rather than back through PMG's own in-process proxy (which would +// re-intercept the request). It honors the process' own proxy environment, +// not the child's injected one. +var goInfoFetchClient = &http.Client{Timeout: 10 * time.Second} + +// fetchPublishTime performs a one-shot authoritative $base/$module/@v/$version.info +// fetch and caches the result. Best-effort: any failure means no publish time. +func (h *goCooldownHandler) fetchPublishTime(ctx *proxy.RequestContext, baseURL, module, version string) (time.Time, bool) { + if baseURL == "" { + return time.Time{}, false + } + + escapedPath, err := gomodule.EscapePath(module) + if err != nil { + return time.Time{}, false + } + + escapedVersion, err := gomodule.EscapeVersion(version) + if err != nil { + return time.Time{}, false + } + + infoURL := fmt.Sprintf("%s/%s/@v/%s.info", strings.TrimSuffix(baseURL, "/"), escapedPath, escapedVersion) + + resp, err := goInfoFetchClient.Get(infoURL) + if err != nil { + log.Warnf("[%s] Cooldown: failed to fetch %s: %v", ctx.RequestID, infoURL, err) + return time.Time{}, false + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + log.Warnf("[%s] Cooldown: fetching %s returned HTTP %d", ctx.RequestID, infoURL, resp.StatusCode) + return time.Time{}, false + } + + var info struct { + Time time.Time `json:"Time"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&info); err != nil || info.Time.IsZero() { + log.Warnf("[%s] Cooldown: failed to parse publish time from %s", ctx.RequestID, infoURL) + return time.Time{}, false + } + + h.mu.Lock() + h.publishTimes[goModuleVersionKey(module, version)] = info.Time + h.mu.Unlock() + + return info.Time, true +} diff --git a/proxy/interceptors/go_cooldown_test.go b/proxy/interceptors/go_cooldown_test.go new file mode 100644 index 0000000..322c9b8 --- /dev/null +++ b/proxy/interceptors/go_cooldown_test.go @@ -0,0 +1,57 @@ +package interceptors + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/safedep/pmg/proxy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoCooldownCheckZipDownloadSideFetch(t *testing.T) { + publishTime := time.Now().Add(-24 * time.Hour) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/example.com/fresh/@v/v1.1.0.info": + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]string{ + "Version": "v1.1.0", + "Time": publishTime.UTC().Format(time.RFC3339), + }) + require.NoError(t, err) + default: + http.NotFound(w, req) + } + })) + defer server.Close() + + ctx := &proxy.RequestContext{RequestID: "test"} + + t.Run("blocks using out-of-band publish time on cache miss", func(t *testing.T) { + h := newGoCooldownHandler(NewAnalysisStatsCollector()) + + resp, handled := h.CheckZipDownload(ctx, server.URL, "example.com/fresh", "v1.1.0", 7) + require.True(t, handled) + assert.Equal(t, proxy.ActionBlock, resp.Action) + assert.Equal(t, http.StatusForbidden, resp.BlockCode) + }) + + t.Run("fails open when the out-of-band fetch fails", func(t *testing.T) { + h := newGoCooldownHandler(NewAnalysisStatsCollector()) + + _, handled := h.CheckZipDownload(ctx, server.URL, "example.com/unknown", "v9.9.9", 7) + assert.False(t, handled) + }) + + t.Run("fails open without a base URL", func(t *testing.T) { + h := newGoCooldownHandler(NewAnalysisStatsCollector()) + + _, handled := h.CheckZipDownload(ctx, "", "example.com/fresh", "v1.1.0", 7) + assert.False(t, handled) + }) +} diff --git a/proxy/interceptors/go_registry.go b/proxy/interceptors/go_registry.go new file mode 100644 index 0000000..2724a87 --- /dev/null +++ b/proxy/interceptors/go_registry.go @@ -0,0 +1,205 @@ +package interceptors + +import ( + "net/url" + "strings" + "sync" + + packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1" + "github.com/safedep/dry/log" + "github.com/safedep/pmg/analyzer" + pmgconfig "github.com/safedep/pmg/config" + "github.com/safedep/pmg/proxy" +) + +// goToolchainModule is the module path Go uses to auto-download toolchains +// (GOTOOLCHAIN=auto). Toolchain zips are verified by go against the checksum +// database regardless of GOPRIVATE/GONOSUMDB, and downloads fail closed when +// GOSUMDB=off, so PMG passes them through on Go's own verification instead of +// treating them as ordinary (never-flagged) modules. +const goToolchainModule = "golang.org/toolchain" + +// GoRegistryInterceptor intercepts Go module proxy requests and analyzes +// module zips for malware. Unlike npm/PyPI, the registry hosts are not fixed: +// they come from the user's effective GOPROXY via +// InterceptorContext.GoProxyBaseURLs. sum.golang.org is never in that set, so +// checksum-database traffic is tunneled, not MITM'd. +type GoRegistryInterceptor struct { + baseRegistryInterceptor + domains registryConfigMap + baseURLs map[string]string + cooldownHandler *goCooldownHandler + + // zipVerdicts memoizes the final response per module zip. go re-requests + // a failed zip (once more during go get's load phase), and without this + // the repeat would double-record stats โ€” the report would show the same + // blocked module twice โ€” and re-prompt the user on a Confirm verdict. + zipVerdictsMu sync.Mutex + zipVerdicts map[string]*proxy.InterceptorResponse +} + +var _ proxy.Interceptor = (*GoRegistryInterceptor)(nil) +var _ proxy.MITMDecider = (*GoRegistryInterceptor)(nil) + +func NewGoRegistryInterceptor( + analyzer analyzer.PackageVersionAnalyzer, + cache AnalysisCache, + statsCollector *AnalysisStatsCollector, + confirmationChan chan *ConfirmationRequest, + execContext InterceptorContext, +) *GoRegistryInterceptor { + domains := registryConfigMap{} + baseURLs := map[string]string{} + for host, baseURL := range execContext.GoProxyBaseURLs { + basePath := "" + if u, err := url.Parse(baseURL); err == nil { + basePath = strings.TrimSuffix(u.Path, "/") + } + + domains[host] = ®istryConfig{ + Host: host, + SupportedForAnalysis: true, + Parser: goProxyParser{basePath: basePath}, + } + baseURLs[host] = baseURL + } + + return &GoRegistryInterceptor{ + baseRegistryInterceptor: baseRegistryInterceptor{ + analyzer: analyzer, + cache: cache, + statsCollector: statsCollector, + confirmationChan: confirmationChan, + circuitBreaker: newAnalyzerCircuitBreaker("malysis-analyzer-go"), + execContext: execContext, + }, + domains: domains, + baseURLs: baseURLs, + cooldownHandler: newGoCooldownHandler(statsCollector), + zipVerdicts: map[string]*proxy.InterceptorResponse{}, + } +} + +func (i *GoRegistryInterceptor) Name() string { + return "go-registry-interceptor" +} + +func (i *GoRegistryInterceptor) ShouldMITM(ctx *proxy.RequestContext) bool { + config := i.domains.GetConfigForHostname(ctx.Hostname) + if config == nil { + return false + } + + return config.SupportedForAnalysis +} + +func (i *GoRegistryInterceptor) ShouldIntercept(ctx *proxy.RequestContext) bool { + return i.domains.ContainsHostname(ctx.Hostname) +} + +// HandleRequest processes the request and returns response action. +// We take a fail-open approach here, allowing requests that we can't parse the +// package information from the URL โ€” but an unparseable .zip means an +// unanalyzed source download, so that case is logged loudly. +func (i *GoRegistryInterceptor) HandleRequest(ctx *proxy.RequestContext) (*proxy.InterceptorResponse, error) { + log.Debugf("[%s] Handling Go module proxy request: %s", ctx.RequestID, ctx.URL.Path) + + config := i.domains.GetConfigForHostname(ctx.Hostname) + if config == nil { + log.Warnf("[%s] No registry config found for hostname: %s", ctx.RequestID, ctx.Hostname) + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + pkgInfo, err := config.Parser.ParseURL(ctx.URL.Path) + if err != nil { + if strings.HasSuffix(ctx.URL.Path, ".zip") { + log.Warnf("[%s] Failed to parse Go module proxy zip URL %s: %v โ€” download allowed without analysis", + ctx.RequestID, ctx.URL.Path, err) + } else { + log.Debugf("[%s] Failed to parse Go module proxy URL %s: %v", ctx.RequestID, ctx.URL.Path, err) + } + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + info, ok := pkgInfo.(*goModuleInfo) + if !ok { + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + if info.requestType == goRequestSumDB { + log.Debugf("[%s] Allowing proxied checksum-database request: %s", ctx.RequestID, ctx.URL.Path) + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + if info.name == goToolchainModule { + if info.IsFileDownload() { + log.Infof("[%s] Allowing Go toolchain download %s@%s (verified by Go's checksum database)", + ctx.RequestID, info.name, info.version) + } + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + depCooldownConfig := pmgconfig.Get().Config.DependencyCooldown + + if !info.IsFileDownload() { + if info.requestType == goRequestInfo && info.version != "" && depCooldownConfig.Enabled { + return i.cooldownHandler.HandleInfoRequest(ctx, info.name, info.version) + } + + log.Debugf("[%s] Skipping analysis for metadata request: %s", ctx.RequestID, info.name) + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, nil + } + + key := goModuleVersionKey(info.name, info.version) + + i.zipVerdictsMu.Lock() + memo := i.zipVerdicts[key] + i.zipVerdictsMu.Unlock() + if memo != nil { + log.Debugf("[%s] Reusing verdict for repeated zip request: %s", ctx.RequestID, key) + return memo, nil + } + + resp, memoize, err := i.handleZipDownload(ctx, config, info, depCooldownConfig) + if err != nil { + return resp, err + } + + if memoize { + i.zipVerdictsMu.Lock() + i.zipVerdicts[key] = resp + i.zipVerdictsMu.Unlock() + } + + return resp, nil +} + +// handleZipDownload runs the security controls for a module source download: +// dependency cooldown, then trusted/insecure fast-allow, then malware +// analysis. memoize is false only when the outcome is a fail-open allow after +// an analyzer error, so a retried request gets another chance to be analyzed. +func (i *GoRegistryInterceptor) handleZipDownload( + ctx *proxy.RequestContext, + config *registryConfig, + info *goModuleInfo, + depCooldownConfig pmgconfig.DependencyCooldownConfig, +) (*proxy.InterceptorResponse, bool, error) { + if depCooldownConfig.Enabled { + if resp, handled := i.cooldownHandler.CheckZipDownload(ctx, i.baseURLs[config.Host], info.name, info.version, depCooldownConfig.Days); handled { + return resp, true, nil + } + } + + if resp, ok := i.fastAllow(ctx, packagev1.Ecosystem_ECOSYSTEM_GO, info.name, info.version); ok { + return resp, true, nil + } + + result, err := i.analyzePackage(ctx, packagev1.Ecosystem_ECOSYSTEM_GO, info.name, info.version) + if err != nil { + log.Errorf("[%s] Failed to analyze package %s@%s: %v", ctx.RequestID, info.name, info.version, err) + return &proxy.InterceptorResponse{Action: proxy.ActionAllow}, false, nil + } + + resp, err := i.handleAnalysisResult(ctx, packagev1.Ecosystem_ECOSYSTEM_GO, info.name, info.version, result) + return resp, err == nil, err +} diff --git a/proxy/interceptors/go_registry_test.go b/proxy/interceptors/go_registry_test.go new file mode 100644 index 0000000..9932c44 --- /dev/null +++ b/proxy/interceptors/go_registry_test.go @@ -0,0 +1,43 @@ +package interceptors + +import ( + "testing" + + "github.com/safedep/pmg/proxy" + "github.com/stretchr/testify/assert" +) + +func TestGoRegistryInterceptorHostMatching(t *testing.T) { + interceptor := NewGoRegistryInterceptor(nil, nil, nil, nil, InterceptorContext{ + GoProxyBaseURLs: map[string]string{ + "proxy.golang.org": "https://proxy.golang.org", + "corp.example.com": "https://corp.example.com/goproxy", + }, + }) + + cases := []struct { + hostname string + wantIntercept bool + wantMITM bool + }{ + {"proxy.golang.org", true, true}, + {"corp.example.com", true, true}, + {"sum.golang.org", false, false}, + {"github.com", false, false}, + {"registry.npmjs.org", false, false}, + } + + for _, tc := range cases { + ctx := &proxy.RequestContext{Hostname: tc.hostname} + assert.Equal(t, tc.wantIntercept, interceptor.ShouldIntercept(ctx), "ShouldIntercept(%s)", tc.hostname) + assert.Equal(t, tc.wantMITM, interceptor.ShouldMITM(ctx), "ShouldMITM(%s)", tc.hostname) + } +} + +func TestGoRegistryInterceptorNoHosts(t *testing.T) { + interceptor := NewGoRegistryInterceptor(nil, nil, nil, nil, InterceptorContext{}) + + ctx := &proxy.RequestContext{Hostname: "proxy.golang.org"} + assert.False(t, interceptor.ShouldIntercept(ctx)) + assert.False(t, interceptor.ShouldMITM(ctx)) +} diff --git a/proxy/interceptors/go_url_parser.go b/proxy/interceptors/go_url_parser.go new file mode 100644 index 0000000..b388c72 --- /dev/null +++ b/proxy/interceptors/go_url_parser.go @@ -0,0 +1,122 @@ +package interceptors + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +const ( + goRequestLatest = "latest" + goRequestList = "list" + goRequestInfo = "info" + goRequestMod = "mod" + goRequestZip = "zip" + + // goRequestSumDB is checksum-database traffic proxied through the module + // proxy ($GOPROXY/sumdb/...). It is passed through unmodified so go's + // signature verification stays intact. + goRequestSumDB = "sumdb" +) + +// goModuleInfo is parsed module information from a Go module proxy URL. +type goModuleInfo struct { + name string + version string + requestType string +} + +var _ packageInfo = (*goModuleInfo)(nil) + +func (g *goModuleInfo) GetName() string { return g.name } + +func (g *goModuleInfo) GetVersion() string { return g.version } + +// IsFileDownload is true only for .zip: the single endpoint that downloads +// module source. .info/.mod are fetched for the entire candidate graph during +// version selection, including modules never selected into the build, so +// analyzing them would block builds over code that never lands in the cache. +func (g *goModuleInfo) IsFileDownload() bool { return g.requestType == goRequestZip } + +// goProxyParser parses Go module proxy URLs per the GOPROXY protocol +// (https://go.dev/ref/mod#goproxy-protocol): +// +// //@latest -> latest version metadata +// //@v/list -> version list +// //@v/.info -> version metadata JSON (publish time) +// //@v/.mod -> go.mod file +// //@v/.zip -> module source archive +// /sumdb//... -> proxied checksum-database traffic +// +// Uppercase letters in module path and version arrive escaped as '!'+lowercase +// and are decoded before use as Malysis query keys. +// +// basePath is the path prefix of the GOPROXY entry (e.g. "/goproxy" for +// GOPROXY=https://corp.example.com/goproxy): go sends requests under that +// base, so it is stripped before the module path is parsed. +type goProxyParser struct { + basePath string +} + +var _ registryURLParser = goProxyParser{} + +func (g goProxyParser) ParseURL(urlPath string) (packageInfo, error) { + if g.basePath != "" { + rest, ok := strings.CutPrefix(urlPath, g.basePath) + if !ok || (rest != "" && rest[0] != '/') { + return nil, fmt.Errorf("go proxy URL %q is outside proxy base path %q", urlPath, g.basePath) + } + urlPath = rest + } + + p := strings.TrimPrefix(urlPath, "/") + if p == "" { + return nil, fmt.Errorf("empty go proxy URL path") + } + + if p == "sumdb" || strings.HasPrefix(p, "sumdb/") { + return &goModuleInfo{requestType: goRequestSumDB}, nil + } + + if escaped, ok := strings.CutSuffix(p, "/@latest"); ok { + name, err := module.UnescapePath(escaped) + if err != nil { + return nil, fmt.Errorf("invalid module path in go proxy URL: %w", err) + } + return &goModuleInfo{name: name, requestType: goRequestLatest}, nil + } + + escapedPath, versionPart, ok := strings.Cut(p, "/@v/") + if !ok { + return nil, fmt.Errorf("go proxy URL missing /@v/ or /@latest marker") + } + + name, err := module.UnescapePath(escapedPath) + if err != nil { + return nil, fmt.Errorf("invalid module path in go proxy URL: %w", err) + } + + if versionPart == "list" { + return &goModuleInfo{name: name, requestType: goRequestList}, nil + } + + dot := strings.LastIndex(versionPart, ".") + if dot <= 0 { + return nil, fmt.Errorf("go proxy URL has no version suffix: %q", versionPart) + } + + requestType := versionPart[dot+1:] + switch requestType { + case goRequestInfo, goRequestMod, goRequestZip: + default: + return nil, fmt.Errorf("unrecognized go proxy version suffix: %q", requestType) + } + + version, err := module.UnescapeVersion(versionPart[:dot]) + if err != nil { + return nil, fmt.Errorf("invalid version in go proxy URL: %w", err) + } + + return &goModuleInfo{name: name, version: version, requestType: requestType}, nil +} diff --git a/proxy/interceptors/go_url_parser_test.go b/proxy/interceptors/go_url_parser_test.go new file mode 100644 index 0000000..b9d0ab6 --- /dev/null +++ b/proxy/interceptors/go_url_parser_test.go @@ -0,0 +1,132 @@ +package interceptors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoProxyParserParseURL(t *testing.T) { + cases := []struct { + name string + path string + wantName string + wantVersion string + wantType string + wantIsDownload bool + wantErrContains string + }{ + { + name: "zip download", + path: "/github.com/x/y/@v/v1.2.3.zip", + wantName: "github.com/x/y", + wantVersion: "v1.2.3", + wantType: goRequestZip, + wantIsDownload: true, + }, + { + name: "info metadata", + path: "/github.com/x/y/@v/v1.2.3.info", + wantName: "github.com/x/y", + wantVersion: "v1.2.3", + wantType: goRequestInfo, + }, + { + name: "mod metadata", + path: "/github.com/x/y/@v/v1.2.3.mod", + wantName: "github.com/x/y", + wantVersion: "v1.2.3", + wantType: goRequestMod, + }, + { + name: "version list", + path: "/github.com/x/y/@v/list", + wantName: "github.com/x/y", + wantType: goRequestList, + }, + { + name: "latest metadata", + path: "/github.com/x/y/@latest", + wantName: "github.com/x/y", + wantType: goRequestLatest, + }, + { + name: "case-escaped module path and version are decoded", + path: "/github.com/!burnt!sushi/toml/@v/!v1.0.0-!rc1.zip", + wantName: "github.com/BurntSushi/toml", + wantVersion: "V1.0.0-Rc1", + wantType: goRequestZip, + wantIsDownload: true, + }, + { + name: "pseudo-version", + path: "/example.com/m/@v/v0.0.0-20191109021931-daa7c04131f5.zip", + wantName: "example.com/m", + wantVersion: "v0.0.0-20191109021931-daa7c04131f5", + wantType: goRequestZip, + wantIsDownload: true, + }, + { + name: "incompatible version", + path: "/github.com/x/y/@v/v4.1.2+incompatible.zip", + wantName: "github.com/x/y", + wantVersion: "v4.1.2+incompatible", + wantType: goRequestZip, + wantIsDownload: true, + }, + { + name: "proxied checksum database traffic", + path: "/sumdb/sum.golang.org/lookup/example.com/m@v1.0.0", + wantType: goRequestSumDB, + }, + { + name: "sumdb capability check", + path: "/sumdb/sum.golang.org/supported", + wantType: goRequestSumDB, + }, + { + name: "missing marker", + path: "/github.com/x/y", + wantErrContains: "missing /@v/", + }, + { + name: "unknown suffix", + path: "/github.com/x/y/@v/v1.2.3.tar", + wantErrContains: "unrecognized go proxy version suffix", + }, + { + name: "empty path", + path: "/", + wantErrContains: "empty go proxy URL path", + }, + { + name: "version without suffix", + path: "/github.com/x/y/@v/v123", + wantErrContains: "no version suffix", + }, + } + + parser := goProxyParser{} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + info, err := parser.ParseURL(tc.path) + + if tc.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrContains) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantName, info.GetName()) + assert.Equal(t, tc.wantVersion, info.GetVersion()) + assert.Equal(t, tc.wantIsDownload, info.IsFileDownload()) + + goInfo, ok := info.(*goModuleInfo) + require.True(t, ok) + assert.Equal(t, tc.wantType, goInfo.requestType) + }) + } +} diff --git a/test/proxye2e/analyzer.go b/test/proxye2e/analyzer.go index b6146e2..8a0b133 100644 --- a/test/proxye2e/analyzer.go +++ b/test/proxye2e/analyzer.go @@ -103,6 +103,10 @@ func (r *AnalyzerRecorder) SetPypi(name, version string, v Verdict) { r.set(packagev1.Ecosystem_ECOSYSTEM_PYPI, name, version, v) } +func (r *AnalyzerRecorder) SetGo(name, version string, v Verdict) { + r.set(packagev1.Ecosystem_ECOSYSTEM_GO, name, version, v) +} + func (r *AnalyzerRecorder) set(eco packagev1.Ecosystem, name, version string, v Verdict) { r.mu.Lock() defer r.mu.Unlock() diff --git a/test/proxye2e/driver.go b/test/proxye2e/driver.go index c27bc1e..5b3faf0 100644 --- a/test/proxye2e/driver.go +++ b/test/proxye2e/driver.go @@ -82,6 +82,48 @@ func (d NpmDriver) Install(name, version string) ExecResult { return res } +type GoDriver struct{ h *Harness } + +func (d GoDriver) goProxyURL(modulePath, version, ext string) string { + return fmt.Sprintf("https://proxy.golang.org/%s/@v/%s%s", + goEscapePath(modulePath), goEscapeVersion(version), ext) +} + +// DownloadZipVia fetches a module zip from an arbitrary GOPROXY base URL, +// e.g. a corporate proxy serving under a path prefix. +func (d GoDriver) DownloadZipVia(baseURL, modulePath, version string) RequestOutcome { + return d.h.get(fmt.Sprintf("%s/%s/@v/%s.zip", + baseURL, goEscapePath(modulePath), goEscapeVersion(version)), nil) +} + +func (d GoDriver) FetchInfo(modulePath, version string) RequestOutcome { + return d.h.get(d.goProxyURL(modulePath, version, ".info"), nil) +} + +func (d GoDriver) FetchMod(modulePath, version string) RequestOutcome { + return d.h.get(d.goProxyURL(modulePath, version, ".mod"), nil) +} + +func (d GoDriver) DownloadZip(modulePath, version string) RequestOutcome { + return d.h.get(d.goProxyURL(modulePath, version, ".zip"), nil) +} + +// Install replays go's fetch sequence for a resolved module version: +// .info, then .mod, then the .zip source archive. +func (d GoDriver) Install(modulePath, version string) ExecResult { + res := ExecResult{} + + info := d.FetchInfo(modulePath, version) + res.add(info) + if info.Err != nil || info.StatusCode != 200 { + return res + } + + res.add(d.FetchMod(modulePath, version)) + res.add(d.DownloadZip(modulePath, version)) + return res +} + type PypiDriver struct{ h *Harness } type PypiSimpleFile struct { diff --git a/test/proxye2e/harness.go b/test/proxye2e/harness.go index 7482aaf..c40becf 100644 --- a/test/proxye2e/harness.go +++ b/test/proxye2e/harness.go @@ -83,11 +83,20 @@ func New(t *testing.T, opts ...Option) *Harness { interceptors.NewInMemoryAnalysisCache(), stats, confChan, - interceptors.InterceptorContext{PinnedVersions: o.pinnedVersions}, + interceptors.InterceptorContext{ + PinnedVersions: o.pinnedVersions, + // proxy.golang.org serves at the root of the plain-HTTP mock (also + // the base for out-of-band .info fetches); corp.example.com serves + // under a base path to exercise GOPROXY path-prefix stripping. + GoProxyBaseURLs: map[string]string{ + "proxy.golang.org": registry.goBaseURL(), + "corp.example.com": registry.goBaseURL() + "/goproxy", + }, + }, ) interceptorList := []proxy.Interceptor{interceptors.NewAuditLoggerInterceptor()} - for _, eco := range []packagev1.Ecosystem{packagev1.Ecosystem_ECOSYSTEM_NPM, packagev1.Ecosystem_ECOSYSTEM_PYPI} { + for _, eco := range []packagev1.Ecosystem{packagev1.Ecosystem_ECOSYSTEM_NPM, packagev1.Ecosystem_ECOSYSTEM_PYPI, packagev1.Ecosystem_ECOSYSTEM_GO} { ic, ierr := factory.CreateInterceptor(eco) require.NoError(t, ierr) interceptorList = append(interceptorList, ic) @@ -158,6 +167,7 @@ func (h *Harness) Close() { func (h *Harness) Npm() NpmDriver { return NpmDriver{h: h} } func (h *Harness) Pypi() PypiDriver { return PypiDriver{h: h} } +func (h *Harness) Go() GoDriver { return GoDriver{h: h} } func (h *Harness) Stats() interceptors.AnalysisStats { return h.stats.GetStats() } diff --git a/test/proxye2e/proxye2e_test.go b/test/proxye2e/proxye2e_test.go index 47799a8..eec61be 100644 --- a/test/proxye2e/proxye2e_test.go +++ b/test/proxye2e/proxye2e_test.go @@ -360,3 +360,210 @@ func TestProxyFlow_Pypi(t *testing.T) { }, }) } + +func TestProxyFlow_Go(t *testing.T) { + RunCases(t, []TestCase{ + { + Name: "clean module is analyzed and allowed", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/m", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/m", "v1.0.0", Clean()) + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("example.com/m", "v1.0.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.False(t, res.Blocked()) + assert.Equal(t, 1, h.Analyzer.AnalyzedCount("example.com/m", "v1.0.0")) + assert.True(t, h.Registry.DownloadedGoZip("example.com/m", "v1.0.0")) + assert.GreaterOrEqual(t, h.Stats().AllowedCount, 1) + }, + }, + { + Name: "verified malware is blocked at the zip download", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/evil", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/evil", "v1.0.0", VerifiedMalware()) + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("example.com/evil", "v1.0.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked()) + assert.False(t, h.Registry.DownloadedGoZip("example.com/evil", "v1.0.0"), + "blocked module source must never reach the client") + assert.Len(t, h.BlockedPackages(), 1) + }, + }, + { + Name: "metadata requests are not analyzed", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/m", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + }, + Exec: func(h *Harness) ExecResult { + res := ExecResult{} + res.add(h.Go().FetchInfo("example.com/m", "v1.0.0")) + res.add(h.Go().FetchMod("example.com/m", "v1.0.0")) + return res + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.False(t, res.Blocked()) + assert.Empty(t, h.Analyzer.Calls(), "info/mod metadata must not trigger analysis") + }, + }, + { + Name: "case-escaped module path is decoded before analysis", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "github.com/BurntSushi/toml", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("github.com/BurntSushi/toml", "v1.0.0", VerifiedMalware()) + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("github.com/BurntSushi/toml", "v1.0.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked(), "verdict keyed by decoded module path must match") + assert.Equal(t, 1, h.Analyzer.AnalyzedCount("github.com/BurntSushi/toml", "v1.0.0")) + }, + }, + { + Name: "suspicious module blocked when user declines", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/maybe", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/maybe", "v1.0.0", Suspicious()) + h.Confirm.AutoDeny() + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("example.com/maybe", "v1.0.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked()) + assert.Len(t, h.Confirm.Prompts(), 1) + assert.False(t, h.Registry.DownloadedGoZip("example.com/maybe", "v1.0.0")) + }, + }, + { + Name: "cooldown blocks in-window version at the zip download", + Config: cooldownEnabled(7), + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/fresh", + Versions: []GoVersion{{Version: "v1.1.0", PublishedAt: recent()}}}) + h.Analyzer.SetGo("example.com/fresh", "v1.1.0", Clean()) + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("example.com/fresh", "v1.1.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked()) + assert.False(t, h.Registry.DownloadedGoZip("example.com/fresh", "v1.1.0")) + assert.GreaterOrEqual(t, h.Stats().CooldownBlockedCount, 1) + + var found bool + for _, b := range h.CooldownBlocks() { + if b.Name == "example.com/fresh" && b.Version == "v1.1.0" { + found = true + } + } + assert.True(t, found, "in-window version should be recorded as a cooldown block") + }, + }, + { + Name: "cooldown allows out-of-window version", + Config: cooldownEnabled(7), + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/m", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/m", "v1.0.0", Clean()) + }, + Exec: func(h *Harness) ExecResult { return h.Go().Install("example.com/m", "v1.0.0") }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.False(t, res.Blocked()) + assert.True(t, h.Registry.DownloadedGoZip("example.com/m", "v1.0.0")) + assert.Equal(t, 0, h.Stats().CooldownBlockedCount) + }, + }, + { + Name: "cooldown side-fetches publish time when .info was cached locally", + Config: cooldownEnabled(7), + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/fresh", + Versions: []GoVersion{{Version: "v1.1.0", PublishedAt: recent()}}}) + h.Analyzer.SetGo("example.com/fresh", "v1.1.0", Clean()) + }, + Exec: func(h *Harness) ExecResult { + // Zip fetched without a prior .info through the proxy (go + // served .info from its local module cache): the interceptor + // must fetch the publish time out-of-band and still block. + res := ExecResult{} + res.add(h.Go().DownloadZip("example.com/fresh", "v1.1.0")) + return res + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked(), "cooldown must block via the out-of-band .info fetch") + assert.GreaterOrEqual(t, h.Stats().CooldownBlockedCount, 1) + assert.Empty(t, h.Analyzer.Calls(), "blocked before malware analysis") + }, + }, + { + Name: "module served under a GOPROXY base path is analyzed and blocked", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/prefixed", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/prefixed", "v1.0.0", VerifiedMalware()) + }, + Exec: func(h *Harness) ExecResult { + res := ExecResult{} + res.add(h.Go().DownloadZipVia("https://corp.example.com/goproxy", "example.com/prefixed", "v1.0.0")) + return res + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Blocked(), "base path must be stripped so the verdict applies") + assert.Equal(t, 1, h.Analyzer.AnalyzedCount("example.com/prefixed", "v1.0.0")) + }, + }, + { + Name: "repeated zip request for a blocked module records the verdict once", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "example.com/evil", + Versions: []GoVersion{{Version: "v1.0.0", PublishedAt: old()}}}) + h.Analyzer.SetGo("example.com/evil", "v1.0.0", VerifiedMalware()) + }, + Exec: func(h *Harness) ExecResult { + // go re-requests a failed zip during go get's load phase; the + // repeat must not double-count stats or re-run analysis. + res := ExecResult{} + res.add(h.Go().DownloadZip("example.com/evil", "v1.0.0")) + res.add(h.Go().DownloadZip("example.com/evil", "v1.0.0")) + return res + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.True(t, res.Requests[0].Blocked) + assert.True(t, res.Requests[1].Blocked) + assert.Len(t, h.BlockedPackages(), 1, "repeat request must not duplicate the blocked record") + assert.Equal(t, 1, h.Stats().BlockedCount) + assert.Equal(t, 1, h.Analyzer.AnalyzedCount("example.com/evil", "v1.0.0")) + }, + }, + { + Name: "toolchain module is allowed without analysis", + Setup: func(h *Harness) { + h.Registry.AddGoModule(GoModule{Path: "golang.org/toolchain", + Versions: []GoVersion{{Version: "v0.0.1-go1.24.0.linux-amd64", PublishedAt: recent()}}}) + }, + Exec: func(h *Harness) ExecResult { + return h.Go().Install("golang.org/toolchain", "v0.0.1-go1.24.0.linux-amd64") + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.False(t, res.Blocked()) + assert.True(t, h.Registry.DownloadedGoZip("golang.org/toolchain", "v0.0.1-go1.24.0.linux-amd64")) + assert.Empty(t, h.Analyzer.Calls(), "toolchain rides on Go's own checksum-db verification") + }, + }, + { + Name: "proxied checksum-database traffic passes through", + Exec: func(h *Harness) ExecResult { + res := ExecResult{} + res.add(h.get("https://proxy.golang.org/sumdb/sum.golang.org/supported", nil)) + return res + }, + Assert: func(t *testing.T, h *Harness, res ExecResult) { + assert.False(t, res.Blocked()) + assert.Empty(t, h.Analyzer.Calls()) + }, + }, + }) +} diff --git a/test/proxye2e/registry.go b/test/proxye2e/registry.go index 0b49247..456b3a5 100644 --- a/test/proxye2e/registry.go +++ b/test/proxye2e/registry.go @@ -6,9 +6,12 @@ import ( "net" "net/http" "net/http/httptest" + "path" "strings" "sync" "time" + + "golang.org/x/mod/module" ) type NpmVersion struct { @@ -34,6 +37,16 @@ type PypiPackage struct { Versions []PypiVersion } +type GoVersion struct { + Version string + PublishedAt time.Time +} + +type GoModule struct { + Path string + Versions []GoVersion +} + type RecordedRequest struct { Host string Method string @@ -47,22 +60,38 @@ type Registry struct { mu sync.Mutex npm map[string]NpmPackage pypi map[string]PypiPackage + gomod map[string]GoModule requests []RecordedRequest server *httptest.Server + goServer *httptest.Server } func newRegistry() *Registry { r := &Registry{ - npm: map[string]NpmPackage{}, - pypi: map[string]PypiPackage{}, + npm: map[string]NpmPackage{}, + pypi: map[string]PypiPackage{}, + gomod: map[string]GoModule{}, } r.server = httptest.NewTLSServer(http.HandlerFunc(r.serve)) + // Plain-HTTP GOPROXY endpoint for the interceptor's out-of-band .info + // fetches, which go straight to the upstream base URL rather than through + // the proxy under test. It also serves the /goproxy base path used to + // exercise GOPROXY path-prefix handling. + r.goServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + r.record(req) + r.serveGoWithOptionalPrefix(w, req) + })) return r } func (r *Registry) addr() string { return r.server.Listener.Addr().String() } -func (r *Registry) close() { r.server.Close() } +func (r *Registry) goBaseURL() string { return r.goServer.URL } + +func (r *Registry) close() { + r.server.Close() + r.goServer.Close() +} func (r *Registry) AddNpm(pkg NpmPackage) { r.mu.Lock() @@ -76,6 +105,12 @@ func (r *Registry) AddPypi(pkg PypiPackage) { r.pypi[normalizePypiName(pkg.Name)] = pkg } +func (r *Registry) AddGoModule(mod GoModule) { + r.mu.Lock() + defer r.mu.Unlock() + r.gomod[mod.Path] = mod +} + // Requests returns every request the proxy forwarded upstream, in order. func (r *Registry) Requests() []RecordedRequest { r.mu.Lock() @@ -97,25 +132,138 @@ func (r *Registry) DownloadedTarball(name, version string) bool { return false } -func (r *Registry) serve(w http.ResponseWriter, req *http.Request) { - host := hostOnly(req.Host) - +func (r *Registry) record(req *http.Request) { r.mu.Lock() - r.requests = append(r.requests, RecordedRequest{Host: host, Method: req.Method, Path: req.URL.Path}) - r.mu.Unlock() + defer r.mu.Unlock() + r.requests = append(r.requests, RecordedRequest{Host: hostOnly(req.Host), Method: req.Method, Path: req.URL.Path}) +} - switch host { +func (r *Registry) serve(w http.ResponseWriter, req *http.Request) { + r.record(req) + + switch hostOnly(req.Host) { case "registry.npmjs.org", "registry.yarnpkg.com": r.serveNpm(w, req) case "pypi.org": r.servePypiSimple(w, req) case "files.pythonhosted.org": r.servePypiFile(w, req) + case "proxy.golang.org", "corp.example.com": + r.serveGoWithOptionalPrefix(w, req) default: http.NotFound(w, req) } } +// serveGoWithOptionalPrefix serves the GOPROXY protocol either at the root +// (proxy.golang.org) or under the /goproxy base path (corp.example.com and +// the corp base URL of the plain-HTTP go server). +func (r *Registry) serveGoWithOptionalPrefix(w http.ResponseWriter, req *http.Request) { + if strings.HasPrefix(req.URL.Path, "/goproxy/") { + http.StripPrefix("/goproxy", http.HandlerFunc(r.serveGo)).ServeHTTP(w, req) + return + } + r.serveGo(w, req) +} + +// DownloadedGoZip reports whether the module zip for the given path and +// version was fetched from the registry. +func (r *Registry) DownloadedGoZip(modulePath, version string) bool { + want := "/" + goEscapePath(modulePath) + "/@v/" + goEscapeVersion(version) + ".zip" + for _, req := range r.Requests() { + if req.Path == want { + return true + } + } + return false +} + +// serveGo implements a minimal GOPROXY protocol endpoint: .info (with publish +// time), .mod and .zip per registered module version, plus /sumdb/* which the +// real proxy serves for checksum-database lookups. +func (r *Registry) serveGo(w http.ResponseWriter, req *http.Request) { + p := strings.TrimPrefix(req.URL.Path, "/") + + if strings.HasPrefix(p, "sumdb/") { + _, _ = w.Write([]byte("e2e-sumdb")) + return + } + + escapedPath, versionPart, found := strings.Cut(p, "/@v/") + if !found { + http.NotFound(w, req) + return + } + + modulePath, err := module.UnescapePath(escapedPath) + if err != nil { + http.NotFound(w, req) + return + } + + r.mu.Lock() + mod, ok := r.gomod[modulePath] + r.mu.Unlock() + if !ok { + http.NotFound(w, req) + return + } + + ext := path.Ext(versionPart) + version, err := module.UnescapeVersion(strings.TrimSuffix(versionPart, ext)) + if err != nil { + http.NotFound(w, req) + return + } + + var published time.Time + versionFound := false + for _, v := range mod.Versions { + if v.Version == version { + published = v.PublishedAt + versionFound = true + break + } + } + if !versionFound { + http.NotFound(w, req) + return + } + + switch ext { + case ".info": + w.Header().Set("Content-Type", "application/json") + body, _ := json.Marshal(map[string]string{ + "Version": version, + "Time": published.UTC().Format(time.RFC3339), + }) + _, _ = w.Write(body) + case ".mod": + _, _ = fmt.Fprintf(w, "module %s\n", modulePath) + case ".zip": + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write([]byte("e2e-module-zip")) + default: + http.NotFound(w, req) + } +} + +func goEscapePath(p string) string { + escaped, err := module.EscapePath(p) + if err != nil { + return p + } + return escaped +} + +func goEscapeVersion(v string) string { + escaped, err := module.EscapeVersion(v) + if err != nil { + return v + } + return escaped +} + func (r *Registry) serveNpm(w http.ResponseWriter, req *http.Request) { path := strings.Trim(req.URL.Path, "/")