Add experimental Go module proxy support (#358)

* feat: add experimental Go module support via pmg go Adds Go modules as a proxy-guarded ecosystem, opt-in only: the command runs solely when invoked explicitly as `pmg go ...` and is deliberately excluded from setup aliases and PATH shims so existing users are unaffected. - packagemanager: goPackageManager with fail-safe command classification (vet/fix excluded from non-download since they can fetch on a cold cache) and pinned-version extraction where only canonical semver counts as explicit. - GOPROXY normalization (fail-closed): effective GOPROXY read via `go env` (honors go env -w), rebuilt comma-joined with `direct` dropped so a 403 block is terminal and nothing silently falls back to unanalyzed VCS fetches. GOPRIVATE/GONOPROXY surface a warning; GOINSECURE is cleared. Contributed to the proxy flow through a new ProxyRoutingProvider hook (extra child env + dynamic MITM hosts). - Go interceptor with dynamic host matching from the user's effective GOPROXY via InterceptorContext.GoProxyHosts. Malware analysis runs on .zip only (the sole endpoint that delivers code); .info/.mod/@latest/ list pass through; /sumdb/ traffic and sum.golang.org are never touched so checksum-db verification stays intact; golang.org/toolchain is allowed on Go's own checksum verification. - Dependency cooldown: publish time captured from .info responses (body unmodified), in-window .zip blocked with 403; fails open for cooldown only when the publish time was never observed. - Cert gate: on macOS/Windows `pmg go` fails fast with actionable guidance unless the persisted PMG CA is OS-trusted (Go ignores SSL_CERT_FILE there); Linux works via the injected bundle. - proxye2e: GOPROXY-protocol mock registry, Go driver and 10 hermetic cases (allow/block/confirm, case-escaped paths, cooldown block and fail-open, toolchain, sumdb passthrough). Verified end-to-end on Linux: `pmg go get github.com/google/uuid@v1.6.0` MITMs proxy.golang.org, analyzes the decoded module at the .zip fetch, and go.sum verification succeeds through the tunneled checksum db. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014xuhBeTVpfU4SdqVaarvuK * fix(go): address review findings on experimental Go support - Drop fmt/clean from NonDownloadCommands: both load packages via go list and can download modules on a cold cache, which would bypass the proxy under install_only. - Support GOPROXY entries with a base path (e.g. corp Athens/JFrog at https://corp/goproxy): the interceptor now receives host -> base URL and strips the path prefix before parsing module URLs, so verdicts and cooldown key on the real module path. - Default unschemed GOPROXY entries to https, matching go's own behavior, so corp mirrors configured as bare hosts are intercepted instead of silently unanalyzed. - Memoize the final verdict per module zip: go re-requests a failed zip during go get's load phase, which double-recorded stats (the report showed the same blocked module twice) and would have re-prompted on Confirm verdicts. - Fetch .info out-of-band on a cooldown cache miss: go serves .info from its local module cache on any machine that used go before PMG, which silently disabled cooldown. Failure of the side-fetch still fails open for cooldown only. - Move the noop package resolver into packagemanager. Verified live: cold-cache cooldown block now records once; warm-cache rerun is blocked via the side-fetch instead of failing open. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014xuhBeTVpfU4SdqVaarvuK * docs: collapse Go proxy-mode details by default Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014xuhBeTVpfU4SdqVaarvuK --------- Co-authored-by: Claude <noreply@anthropic.com>
2026-08-03 07:24:09 +02:00 · 2026-07-03 18:19:31 +05:30
parent 648adcbda4
commit b19473945b
24 changed files with 1941 additions and 15 deletions
@@ -6,9 +6,12 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"path"
 	"strings"
 	"sync"
 	"time"
+
+	"golang.org/x/mod/module"
 )

 type NpmVersion struct {
@@ -34,6 +37,16 @@ type PypiPackage struct {
 	Versions []PypiVersion
 }

+type GoVersion struct {
+	Version     string
+	PublishedAt time.Time
+}
+
+type GoModule struct {
+	Path     string
+	Versions []GoVersion
+}
+
 type RecordedRequest struct {
 	Host   string
 	Method string
@@ -47,22 +60,38 @@ type Registry struct {
 	mu       sync.Mutex
 	npm      map[string]NpmPackage
 	pypi     map[string]PypiPackage
+	gomod    map[string]GoModule
 	requests []RecordedRequest
 	server   *httptest.Server
+	goServer *httptest.Server
 }

 func newRegistry() *Registry {
 	r := &Registry{
-		npm:  map[string]NpmPackage{},
-		pypi: map[string]PypiPackage{},
+		npm:   map[string]NpmPackage{},
+		pypi:  map[string]PypiPackage{},
+		gomod: map[string]GoModule{},
 	}
 	r.server = httptest.NewTLSServer(http.HandlerFunc(r.serve))
+	// Plain-HTTP GOPROXY endpoint for the interceptor's out-of-band .info
+	// fetches, which go straight to the upstream base URL rather than through
+	// the proxy under test. It also serves the /goproxy base path used to
+	// exercise GOPROXY path-prefix handling.
+	r.goServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		r.record(req)
+		r.serveGoWithOptionalPrefix(w, req)
+	}))
 	return r
 }

 func (r *Registry) addr() string { return r.server.Listener.Addr().String() }

-func (r *Registry) close() { r.server.Close() }
+func (r *Registry) goBaseURL() string { return r.goServer.URL }
+
+func (r *Registry) close() {
+	r.server.Close()
+	r.goServer.Close()
+}

 func (r *Registry) AddNpm(pkg NpmPackage) {
 	r.mu.Lock()
@@ -76,6 +105,12 @@ func (r *Registry) AddPypi(pkg PypiPackage) {
 	r.pypi[normalizePypiName(pkg.Name)] = pkg
 }

+func (r *Registry) AddGoModule(mod GoModule) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.gomod[mod.Path] = mod
+}
+
 // Requests returns every request the proxy forwarded upstream, in order.
 func (r *Registry) Requests() []RecordedRequest {
 	r.mu.Lock()
@@ -97,25 +132,138 @@ func (r *Registry) DownloadedTarball(name, version string) bool {
 	return false
 }

-func (r *Registry) serve(w http.ResponseWriter, req *http.Request) {
-	host := hostOnly(req.Host)
-
+func (r *Registry) record(req *http.Request) {
 	r.mu.Lock()
-	r.requests = append(r.requests, RecordedRequest{Host: host, Method: req.Method, Path: req.URL.Path})
-	r.mu.Unlock()
+	defer r.mu.Unlock()
+	r.requests = append(r.requests, RecordedRequest{Host: hostOnly(req.Host), Method: req.Method, Path: req.URL.Path})
+}

-	switch host {
+func (r *Registry) serve(w http.ResponseWriter, req *http.Request) {
+	r.record(req)
+
+	switch hostOnly(req.Host) {
 	case "registry.npmjs.org", "registry.yarnpkg.com":
 		r.serveNpm(w, req)
 	case "pypi.org":
 		r.servePypiSimple(w, req)
 	case "files.pythonhosted.org":
 		r.servePypiFile(w, req)
+	case "proxy.golang.org", "corp.example.com":
+		r.serveGoWithOptionalPrefix(w, req)
 	default:
 		http.NotFound(w, req)
 	}
 }

+// serveGoWithOptionalPrefix serves the GOPROXY protocol either at the root
+// (proxy.golang.org) or under the /goproxy base path (corp.example.com and
+// the corp base URL of the plain-HTTP go server).
+func (r *Registry) serveGoWithOptionalPrefix(w http.ResponseWriter, req *http.Request) {
+	if strings.HasPrefix(req.URL.Path, "/goproxy/") {
+		http.StripPrefix("/goproxy", http.HandlerFunc(r.serveGo)).ServeHTTP(w, req)
+		return
+	}
+	r.serveGo(w, req)
+}
+
+// DownloadedGoZip reports whether the module zip for the given path and
+// version was fetched from the registry.
+func (r *Registry) DownloadedGoZip(modulePath, version string) bool {
+	want := "/" + goEscapePath(modulePath) + "/@v/" + goEscapeVersion(version) + ".zip"
+	for _, req := range r.Requests() {
+		if req.Path == want {
+			return true
+		}
+	}
+	return false
+}
+
+// serveGo implements a minimal GOPROXY protocol endpoint: .info (with publish
+// time), .mod and .zip per registered module version, plus /sumdb/* which the
+// real proxy serves for checksum-database lookups.
+func (r *Registry) serveGo(w http.ResponseWriter, req *http.Request) {
+	p := strings.TrimPrefix(req.URL.Path, "/")
+
+	if strings.HasPrefix(p, "sumdb/") {
+		_, _ = w.Write([]byte("e2e-sumdb"))
+		return
+	}
+
+	escapedPath, versionPart, found := strings.Cut(p, "/@v/")
+	if !found {
+		http.NotFound(w, req)
+		return
+	}
+
+	modulePath, err := module.UnescapePath(escapedPath)
+	if err != nil {
+		http.NotFound(w, req)
+		return
+	}
+
+	r.mu.Lock()
+	mod, ok := r.gomod[modulePath]
+	r.mu.Unlock()
+	if !ok {
+		http.NotFound(w, req)
+		return
+	}
+
+	ext := path.Ext(versionPart)
+	version, err := module.UnescapeVersion(strings.TrimSuffix(versionPart, ext))
+	if err != nil {
+		http.NotFound(w, req)
+		return
+	}
+
+	var published time.Time
+	versionFound := false
+	for _, v := range mod.Versions {
+		if v.Version == version {
+			published = v.PublishedAt
+			versionFound = true
+			break
+		}
+	}
+	if !versionFound {
+		http.NotFound(w, req)
+		return
+	}
+
+	switch ext {
+	case ".info":
+		w.Header().Set("Content-Type", "application/json")
+		body, _ := json.Marshal(map[string]string{
+			"Version": version,
+			"Time":    published.UTC().Format(time.RFC3339),
+		})
+		_, _ = w.Write(body)
+	case ".mod":
+		_, _ = fmt.Fprintf(w, "module %s\n", modulePath)
+	case ".zip":
+		w.Header().Set("Content-Type", "application/zip")
+		_, _ = w.Write([]byte("e2e-module-zip"))
+	default:
+		http.NotFound(w, req)
+	}
+}
+
+func goEscapePath(p string) string {
+	escaped, err := module.EscapePath(p)
+	if err != nil {
+		return p
+	}
+	return escaped
+}
+
+func goEscapeVersion(v string) string {
+	escaped, err := module.EscapeVersion(v)
+	if err != nil {
+		return v
+	}
+	return escaped
+}
+
 func (r *Registry) serveNpm(w http.ResponseWriter, req *http.Request) {
 	path := strings.Trim(req.URL.Path, "/")