feat: Add malysis cache implementation with proxy flow integration (#346)

* feat: Add malysis cache implementation with proxy flow integration

* fix: Code review fixes
This commit is contained in:
Abhisek Datta
2026-06-22 10:12:02 +05:30
committed by GitHub
parent 327c9c7068
commit d360e75897
21 changed files with 820 additions and 21 deletions
+40
View File
@@ -0,0 +1,40 @@
package analyzer
import (
"context"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/safedep/dry/log"
)
// malysisCachingAnalyzer is a read-through cache decorator over a PackageVersionAnalyzer.
type malysisCachingAnalyzer struct {
PackageVersionAnalyzer
cache MalysisCache
}
func newMalysisCachingAnalyzer(next PackageVersionAnalyzer, cache MalysisCache) *malysisCachingAnalyzer {
return &malysisCachingAnalyzer{PackageVersionAnalyzer: next, cache: cache}
}
func (c *malysisCachingAnalyzer) Analyze(ctx context.Context, pkg *packagev1.PackageVersion) (*PackageVersionAnalysisResult, error) {
name, version := pkg.GetPackage().GetName(), pkg.GetVersion()
if result, ok, err := c.cache.Get(ctx, pkg); err != nil {
log.Warnf("malysis cache lookup failed, falling back to live analysis: %v", err)
} else if ok {
log.Debugf("malysis cache hit: %s@%s", name, version)
return result, nil
}
log.Debugf("malysis cache miss: %s@%s", name, version)
result, err := c.PackageVersionAnalyzer.Analyze(ctx, pkg)
if err != nil {
return nil, err
}
if err := c.cache.Set(ctx, pkg, result); err != nil {
log.Warnf("malysis cache store failed: %v", err)
}
return result, nil
}
+85
View File
@@ -0,0 +1,85 @@
package analyzer
import (
"context"
"errors"
"testing"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type fakeCache struct {
getResult *PackageVersionAnalysisResult
getOK bool
getErr error
setErr error
setCalls int
}
func (f *fakeCache) Get(context.Context, *packagev1.PackageVersion) (*PackageVersionAnalysisResult, bool, error) {
return f.getResult, f.getOK, f.getErr
}
func (f *fakeCache) Set(context.Context, *packagev1.PackageVersion, *PackageVersionAnalysisResult) error {
f.setCalls++
return f.setErr
}
type fakeAnalyzer struct {
result *PackageVersionAnalysisResult
err error
callCount int
}
func (f *fakeAnalyzer) Name() string { return "fake" }
func (f *fakeAnalyzer) Analyze(context.Context, *packagev1.PackageVersion) (*PackageVersionAnalysisResult, error) {
f.callCount++
return f.result, f.err
}
func TestMalysisCachingAnalyzer_Hit(t *testing.T) {
cached := &PackageVersionAnalysisResult{Action: ActionAllow, AnalysisID: "cached"}
next := &fakeAnalyzer{}
a := newMalysisCachingAnalyzer(next, &fakeCache{getResult: cached, getOK: true})
got, err := a.Analyze(context.Background(), &packagev1.PackageVersion{})
require.NoError(t, err)
assert.Equal(t, "cached", got.AnalysisID)
assert.Equal(t, 0, next.callCount, "hit must not delegate")
}
func TestMalysisCachingAnalyzer_MissDelegatesAndSets(t *testing.T) {
live := &PackageVersionAnalysisResult{Action: ActionAllow, AnalysisID: "live"}
next := &fakeAnalyzer{result: live}
fc := &fakeCache{}
a := newMalysisCachingAnalyzer(next, fc)
got, err := a.Analyze(context.Background(), &packagev1.PackageVersion{})
require.NoError(t, err)
assert.Equal(t, "live", got.AnalysisID)
assert.Equal(t, 1, next.callCount)
assert.Equal(t, 1, fc.setCalls)
}
func TestMalysisCachingAnalyzer_FailSoft(t *testing.T) {
live := &PackageVersionAnalysisResult{Action: ActionAllow, AnalysisID: "live"}
next := &fakeAnalyzer{result: live}
// Get error => treat as miss and delegate; Set error => still return result.
a := newMalysisCachingAnalyzer(next, &fakeCache{getErr: errors.New("boom"), setErr: errors.New("boom")})
got, err := a.Analyze(context.Background(), &packagev1.PackageVersion{})
require.NoError(t, err)
assert.Equal(t, "live", got.AnalysisID)
assert.Equal(t, 1, next.callCount)
}
func TestMalysisCachingAnalyzer_AnalyzeErrorNotCached(t *testing.T) {
next := &fakeAnalyzer{err: errors.New("network")}
fc := &fakeCache{}
a := newMalysisCachingAnalyzer(next, fc)
_, err := a.Analyze(context.Background(), &packagev1.PackageVersion{})
require.Error(t, err)
assert.Equal(t, 0, fc.setCalls, "errored analysis must not be cached")
}
+10 -2
View File
@@ -28,7 +28,7 @@ func newMalysisAnalyzer(config MalysisQueryAnalyzerConfig,
creds, closeResolver, err := resolveCredentials()
if err != nil {
log.Debugf("SafeDep Cloud credentials unavailable, using community malysis analyzer: %v", err)
return community, nil
return withCache(community, config), nil
}
defer func() {
if closeErr := closeResolver(); closeErr != nil {
@@ -42,5 +42,13 @@ func newMalysisAnalyzer(config MalysisQueryAnalyzerConfig,
}
log.Debugf("SafeDep Cloud credentials found, using authenticated malysis analyzer with community fallback")
return newMalysisFallbackAnalyzer(authenticated, community), nil
return withCache(newMalysisFallbackAnalyzer(authenticated, community), config), nil
}
// withCache wraps a in a read-through cache decorator when one is configured.
func withCache(a PackageVersionAnalyzer, config MalysisQueryAnalyzerConfig) PackageVersionAnalyzer {
if config.Cache == nil {
return a
}
return newMalysisCachingAnalyzer(a, config.Cache)
}
+5 -1
View File
@@ -21,7 +21,11 @@ const (
communityMalysisPort = "443"
)
type MalysisQueryAnalyzerConfig struct{}
type MalysisQueryAnalyzerConfig struct {
// Cache, when non-nil, enables a persistent read-through verdict cache
// applied as a decorator by newMalysisAnalyzer. nil = no caching.
Cache MalysisCache
}
type malysisQueryAnalyzer struct {
client malysisv1grpc.MalwareAnalysisServiceClient
+149
View File
@@ -0,0 +1,149 @@
// Package malysiscache is a localdb-backed persistent cache of benign Malysis
// verdicts. It implements analyzer.MalysisCache. It owns only its localdb
// module schema; the DB file location is owned by the config package and the
// Manager lifecycle by the composition root.
package malysiscache
import (
"context"
"database/sql"
"errors"
"time"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/safedep/dry/localdb"
"github.com/safedep/dry/log"
"github.com/safedep/pmg/analyzer"
"github.com/safedep/pmg/config"
)
var _ analyzer.MalysisCache = (*Cache)(nil)
const moduleName = "malysis_cache"
// Descriptor is the localdb module contract: the verdicts table. Migrations are
// append-only — never edit or reorder an existing entry.
func Descriptor() localdb.Descriptor {
return localdb.Descriptor{
Name: moduleName,
Migrations: []string{
`CREATE TABLE malysis_cache_verdicts (
ecosystem TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
analysis_id TEXT,
reference_url TEXT,
summary TEXT,
created_at INTEGER NOT NULL,
expires_at INTEGER,
PRIMARY KEY (ecosystem, name, version)
)`,
},
}
}
type Cache struct {
db *sql.DB
cfg config.MalysisCacheConfig
now func() time.Time
}
func New(store *localdb.Store, cfg config.MalysisCacheConfig) *Cache {
return &Cache{db: store.DB(), cfg: cfg, now: time.Now}
}
type Stats struct {
Count int
Oldest time.Time
Newest time.Time
}
func (c *Cache) Stats(ctx context.Context) (Stats, error) {
var s Stats
var oldest, newest sql.NullInt64
err := c.db.QueryRowContext(ctx,
`SELECT COUNT(*), MIN(created_at), MAX(created_at) FROM malysis_cache_verdicts`).
Scan(&s.Count, &oldest, &newest)
if err != nil {
return Stats{}, err
}
if oldest.Valid {
s.Oldest = time.Unix(oldest.Int64, 0)
}
if newest.Valid {
s.Newest = time.Unix(newest.Int64, 0)
}
return s, nil
}
func (c *Cache) Clear(ctx context.Context) error {
_, err := c.db.ExecContext(ctx, `DELETE FROM malysis_cache_verdicts`)
return err
}
func (c *Cache) Get(ctx context.Context, pkg *packagev1.PackageVersion) (*analyzer.PackageVersionAnalysisResult, bool, error) {
eco, name, version := packageKey(pkg)
var analysisID, referenceURL, summary string
var createdAt int64
var expiresAt sql.NullInt64
err := c.db.QueryRowContext(ctx,
`SELECT analysis_id, reference_url, summary, created_at, expires_at
FROM malysis_cache_verdicts WHERE ecosystem=? AND name=? AND version=?`,
eco, name, version).
Scan(&analysisID, &referenceURL, &summary, &createdAt, &expiresAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
if c.expired(createdAt, expiresAt) {
if _, derr := c.db.ExecContext(ctx,
`DELETE FROM malysis_cache_verdicts WHERE ecosystem=? AND name=? AND version=?`,
eco, name, version); derr != nil {
log.Warnf("malysiscache: failed to delete expired entry: %v", derr)
}
return nil, false, nil
}
result, ok := reconstruct(eco, name, version, analysisID, referenceURL, summary)
if !ok {
return nil, false, nil
}
return result, true, nil
}
func (c *Cache) Set(ctx context.Context, pkg *packagev1.PackageVersion, result *analyzer.PackageVersionAnalysisResult) error {
if !cacheable(result) {
return nil
}
// Non-positive TTL disables persistence (MalysisCacheConfig.TTL contract).
if c.cfg.TTL <= 0 {
return nil
}
eco, name, version := packageKey(pkg)
// expires_at stays NULL in v1; a future backend hint populates it.
_, err := c.db.ExecContext(ctx,
`INSERT INTO malysis_cache_verdicts
(ecosystem, name, version, analysis_id, reference_url, summary, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, NULL)
ON CONFLICT(ecosystem, name, version) DO UPDATE SET
analysis_id = excluded.analysis_id,
reference_url = excluded.reference_url,
summary = excluded.summary,
created_at = excluded.created_at,
expires_at = excluded.expires_at`,
eco, name, version, result.AnalysisID, result.ReferenceURL, result.Summary, c.now().Unix())
return err
}
func (c *Cache) expired(createdAt int64, expiresAt sql.NullInt64) bool {
exp := createdAt + int64(c.cfg.TTL.Seconds())
if expiresAt.Valid {
exp = expiresAt.Int64
}
return c.now().Unix() >= exp
}
+108
View File
@@ -0,0 +1,108 @@
package malysiscache
import (
"context"
"testing"
"time"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/safedep/dry/localdb"
"github.com/safedep/pmg/analyzer"
"github.com/safedep/pmg/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newTestCache(t *testing.T) *Cache {
t.Helper()
mgr := localdb.New(localdb.Config{Dir: t.TempDir(), FileName: "pmg.db"})
t.Cleanup(func() { require.NoError(t, mgr.Close()) })
store, err := mgr.Store(context.Background(), Descriptor())
require.NoError(t, err)
return New(store, config.MalysisCacheConfig{TTL: 24 * time.Hour})
}
func TestStatsAndClearEmpty(t *testing.T) {
c := newTestCache(t)
s, err := c.Stats(context.Background())
require.NoError(t, err)
assert.Equal(t, 0, s.Count)
assert.True(t, s.Oldest.IsZero())
require.NoError(t, c.Clear(context.Background()))
}
func allow(name, version string) *analyzer.PackageVersionAnalysisResult {
return &analyzer.PackageVersionAnalysisResult{
Action: analyzer.ActionAllow,
AnalysisID: "aid-" + version,
ReferenceURL: "https://ref/" + version,
Summary: "clean",
}
}
func TestSetGetRoundTrip(t *testing.T) {
c := newTestCache(t)
ctx := context.Background()
p := pkg(packagev1.Ecosystem_ECOSYSTEM_NPM, "left-pad", "1.0.0")
require.NoError(t, c.Set(ctx, p, allow("left-pad", "1.0.0")))
got, ok, err := c.Get(ctx, p)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, "aid-1.0.0", got.AnalysisID)
assert.Equal(t, analyzer.ActionAllow, got.Action)
assert.Equal(t, "left-pad", got.PackageVersion.GetPackage().GetName())
}
func TestSetSkipsWriteWhenTTLNonPositive(t *testing.T) {
mgr := localdb.New(localdb.Config{Dir: t.TempDir(), FileName: "pmg.db"})
t.Cleanup(func() { require.NoError(t, mgr.Close()) })
store, err := mgr.Store(context.Background(), Descriptor())
require.NoError(t, err)
c := New(store, config.MalysisCacheConfig{TTL: 0})
ctx := context.Background()
p := pkg(packagev1.Ecosystem_ECOSYSTEM_NPM, "left-pad", "1.0.0")
require.NoError(t, c.Set(ctx, p, allow("left-pad", "1.0.0")))
s, err := c.Stats(ctx)
require.NoError(t, err)
assert.Equal(t, 0, s.Count, "non-positive TTL disables persistence: nothing written")
}
func TestSetSkipsNonCacheable(t *testing.T) {
c := newTestCache(t)
ctx := context.Background()
p := pkg(packagev1.Ecosystem_ECOSYSTEM_NPM, "evil", "9.9.9")
excluded := &analyzer.PackageVersionAnalysisResult{Action: analyzer.ActionAllow, IsMalware: true, IsExcluded: true}
require.NoError(t, c.Set(ctx, p, excluded))
_, ok, err := c.Get(ctx, p)
require.NoError(t, err)
assert.False(t, ok, "excluded-malware must never be cached")
}
func TestGetMissAndExpiry(t *testing.T) {
c := newTestCache(t)
ctx := context.Background()
p := pkg(packagev1.Ecosystem_ECOSYSTEM_PYPI, "requests", "2.0.0")
_, ok, err := c.Get(ctx, p)
require.NoError(t, err)
assert.False(t, ok)
require.NoError(t, c.Set(ctx, p, allow("requests", "2.0.0")))
// Advance the clock past the TTL: the entry is expired and lazily deleted.
c.now = func() time.Time { return time.Now().Add(25 * time.Hour) }
_, ok, err = c.Get(ctx, p)
require.NoError(t, err)
assert.False(t, ok)
s, err := c.Stats(ctx)
require.NoError(t, err)
assert.Equal(t, 0, s.Count, "expired row should be lazily deleted")
}
+59
View File
@@ -0,0 +1,59 @@
package malysiscache
import (
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/safedep/pmg/analyzer"
)
// cacheable is the security predicate: only a clean benign allow is ever
// persisted. ActionAllow alone is NOT sufficient — applyExclusion downgrades a
// confirmed-malicious package to ActionAllow while leaving IsMalware=true and
// setting IsExcluded=true. A tenant exclusion is auth-scoped and revocable, not
// a property of the artifact, so it must never be cached.
func cacheable(r *analyzer.PackageVersionAnalysisResult) bool {
return r != nil && r.Action == analyzer.ActionAllow && !r.IsMalware && !r.IsExcluded
}
// packageKey extracts the verbatim cache key. No canonicalization: the cache
// keys on whatever identity the pipeline already uses.
func packageKey(pkg *packagev1.PackageVersion) (eco, name, version string) {
return pkg.GetPackage().GetEcosystem().String(),
pkg.GetPackage().GetName(),
pkg.GetVersion()
}
// ecosystemFromString maps the stored enum name back to the enum. ok is false
// for an unknown name (a row written by a newer binary that knows an ecosystem
// this one does not).
func ecosystemFromString(s string) (packagev1.Ecosystem, bool) {
v, ok := packagev1.Ecosystem_value[s]
if !ok {
return packagev1.Ecosystem_ECOSYSTEM_UNSPECIFIED, false
}
return packagev1.Ecosystem(v), true
}
// reconstruct rebuilds a cached ALLOW verdict from a row. ok is false when the
// ecosystem name is unknown, so the caller treats it as a miss. The malware /
// exclusion fields are hard-coded false: cacheable() guarantees only clean
// benign allows were ever stored.
func reconstruct(eco, name, version, analysisID, referenceURL, summary string) (*analyzer.PackageVersionAnalysisResult, bool) {
ecosystem, ok := ecosystemFromString(eco)
if !ok {
return nil, false
}
pv := &packagev1.PackageVersion{}
pv.SetPackage(&packagev1.Package{})
pv.GetPackage().SetName(name)
pv.GetPackage().SetEcosystem(ecosystem)
pv.SetVersion(version)
return &analyzer.PackageVersionAnalysisResult{
PackageVersion: pv,
AnalysisID: analysisID,
ReferenceURL: referenceURL,
Action: analyzer.ActionAllow,
Summary: summary,
}, true
}
+56
View File
@@ -0,0 +1,56 @@
package malysiscache
import (
"testing"
packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
"github.com/safedep/pmg/analyzer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func pkg(eco packagev1.Ecosystem, name, version string) *packagev1.PackageVersion {
pv := &packagev1.PackageVersion{}
pv.SetPackage(&packagev1.Package{})
pv.GetPackage().SetName(name)
pv.GetPackage().SetEcosystem(eco)
pv.SetVersion(version)
return pv
}
func TestCacheable(t *testing.T) {
cases := []struct {
name string
in *analyzer.PackageVersionAnalysisResult
want bool
}{
{"clean allow", &analyzer.PackageVersionAnalysisResult{Action: analyzer.ActionAllow}, true},
{"block", &analyzer.PackageVersionAnalysisResult{Action: analyzer.ActionBlock}, false},
{"allow but malware", &analyzer.PackageVersionAnalysisResult{Action: analyzer.ActionAllow, IsMalware: true}, false},
{"excluded malware downgrade", &analyzer.PackageVersionAnalysisResult{Action: analyzer.ActionAllow, IsMalware: true, IsExcluded: true}, false},
{"nil", nil, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, cacheable(tc.in))
})
}
}
func TestKeyAndReconstruct(t *testing.T) {
eco, name, ver := packageKey(pkg(packagev1.Ecosystem_ECOSYSTEM_NPM, "left-pad", "1.0.0"))
assert.Equal(t, "ECOSYSTEM_NPM", eco)
assert.Equal(t, "left-pad", name)
assert.Equal(t, "1.0.0", ver)
res, ok := reconstruct(eco, name, ver, "aid", "https://ref", "ok")
require.True(t, ok)
assert.Equal(t, analyzer.ActionAllow, res.Action)
assert.False(t, res.IsMalware)
assert.Equal(t, packagev1.Ecosystem_ECOSYSTEM_NPM, res.PackageVersion.GetPackage().GetEcosystem())
assert.Equal(t, "left-pad", res.PackageVersion.GetPackage().GetName())
assert.Equal(t, "1.0.0", res.PackageVersion.GetVersion())
_, ok = reconstruct("ECOSYSTEM_FROM_THE_FUTURE", name, ver, "", "", "")
assert.False(t, ok)
}