mirror of
https://github.com/runbear-io/beardrive.git
synced 2026-08-25 08:08:08 +02:00
feat(web): read heatmap phase 1 — ledger, heat API, folder heat dots
Read telemetry per docs/design/read-heatmap.md: a ReadLedger over a new batch-oriented MetaStore ReadRepo (file reads.json + SQL read_stats) aggregates viewer and share reads into daily per-actor buckets, debounced to visits, folded into all-time rows past retention. GET /api/p/<id>/heat serves per-path counts (human/agent/share, distinct readers, last read) — never identities. /store sync traffic and history blob views are not reads. The viewer shows heat dots and read counts on folder listings and the file meta line. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01P5cxPQdSGJnjXCYY9GeWXt
This commit is contained in:
co-authored by
Claude Fable 5
parent
c60018e5cf
commit
28dc882c66
@@ -57,6 +57,13 @@ type webConfig struct {
|
||||
Driver string `json:"driver,omitempty"` // file (default) | sqlite | postgres
|
||||
DSN string `json:"dsn,omitempty"` // sqlite file path, or a Postgres/Supabase URL
|
||||
} `json:"database,omitempty"`
|
||||
// Reads tunes read telemetry (the heat API): aggregate view counts per
|
||||
// file, split human/agent/share. On by default in hub mode; counts only,
|
||||
// no reader identities in any API.
|
||||
Reads *struct {
|
||||
Enabled *bool `json:"enabled,omitempty"` // default true
|
||||
RetentionDays int `json:"retention_days,omitempty"` // default 400; older days fold into all-time
|
||||
} `json:"reads,omitempty"`
|
||||
}
|
||||
|
||||
func loadWebConfig(path string) (webConfig, error) {
|
||||
@@ -340,6 +347,26 @@ credentials); otherwise it is relayed through this server.`,
|
||||
return fmt.Errorf("open share registry: %w", err)
|
||||
}
|
||||
srv.Shares = shares
|
||||
readsOn, retention := true, 0
|
||||
if cfg.Reads != nil {
|
||||
if cfg.Reads.Enabled != nil {
|
||||
readsOn = *cfg.Reads.Enabled
|
||||
}
|
||||
retention = cfg.Reads.RetentionDays
|
||||
}
|
||||
if readsOn {
|
||||
var reads *webapp.ReadLedger
|
||||
if meta != nil {
|
||||
reads, err = webapp.NewReadLedger(meta.Reads(), retention)
|
||||
} else {
|
||||
reads, err = webapp.OpenReadLedger(filepath.Join(filepath.Dir(projectsDB), "reads.json"), retention)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("open read ledger: %w", err)
|
||||
}
|
||||
defer reads.Close()
|
||||
srv.Reads = reads
|
||||
}
|
||||
if meta != nil {
|
||||
display += " (db: " + cfg.Database.Driver + ")"
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
# Read heatmap — design
|
||||
|
||||
Status: proposed (2026-07-11) · Owner: snow · Prior art: session-linked notes (shipped), history API
|
||||
|
||||
## Problem
|
||||
|
||||
BearDrive knows everything about *writes* (journals: who, when, which session)
|
||||
and nothing about *reads*. Admins curating a shared knowledge folder can't see
|
||||
which files the team actually consumes, so they can't tell a load-bearing
|
||||
document from dead weight. The killer view is the **read×write matrix**:
|
||||
heavily-read + long-unwritten is the danger zone (stale knowledge people still
|
||||
rely on); unread + unwritten is archive material.
|
||||
|
||||
Two properties make this more than Confluence-style view counts:
|
||||
|
||||
1. **Write provenance is already perfect** (journals), so read data alone
|
||||
completes the matrix — competitors have reads but weak write history.
|
||||
2. **Agent reads are attributable.** The agent hooks + session notes shipped in
|
||||
PR #12 mean we can distinguish *human* reads from *agent* reads — the agent
|
||||
hot-path is effectively the team's context window, and nobody else can see it.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **`/store/*` sync traffic is never a read.** Devices replicating a volume is
|
||||
replication, not consumption. Only deliberate consumption counts.
|
||||
- No per-user browsing profiles in any API response. Aggregate counts only.
|
||||
- No general web analytics (referrers, dwell time, scroll depth).
|
||||
- Volume mode (`DirSource`, auth-free viewer) is out of scope — like history,
|
||||
this is a hub feature.
|
||||
|
||||
## What counts as a read
|
||||
|
||||
| Source | Kind | Actor (internal only) | Where recorded |
|
||||
|---|---|---|---|
|
||||
| Viewer `GET file` / `render` / `download` | `human` | account email | `serveBlob`, `handleRender` |
|
||||
| Share link hit `GET /s/<token>` | `share` | share token | `handleShared` |
|
||||
| Agent tool read (Read / read_file / …) | `agent` | device id | reported by client, phase 3 |
|
||||
| History version view (`/blob`) | — not counted | | spelunking ≠ consumption |
|
||||
| `/store/*` | — never | | replication |
|
||||
|
||||
Details:
|
||||
|
||||
- **Record before the ETag check.** A 304 render is still a person reading the
|
||||
file; skipping it would undercount exactly the hottest (most-cached) pages.
|
||||
- **Debounce to visits, not requests.** An in-memory `(project, path, actor)`
|
||||
seen-map with a 10-minute window collapses reload storms and the
|
||||
render-then-raw double fetch into one read. The map is pruned on the flush
|
||||
tick. Embedded assets fetched during a markdown render do count as reads of
|
||||
those assets ("this diagram is viewed a lot" is signal, not noise).
|
||||
|
||||
## Data model
|
||||
|
||||
One new MetaStore repo. Rows are daily aggregation buckets, not events — the
|
||||
ledger never stores an event log, so there is nothing sensitive to leak and
|
||||
nothing that grows per-request.
|
||||
|
||||
```go
|
||||
// ReadStat is one aggregation bucket: reads of one path by one actor on one
|
||||
// day. Day=="" is the all-time fold (see retention). Actor is an opaque
|
||||
// internal id (account email / device id / share token) used only to count
|
||||
// distinct readers; it never appears in an API response.
|
||||
type ReadStat struct {
|
||||
Project string `json:"project"`
|
||||
Path string `json:"path"`
|
||||
Day string `json:"day"` // "2026-07-11" UTC, or "" for all-time
|
||||
Kind string `json:"kind"` // human | agent | share
|
||||
Actor string `json:"actor"`
|
||||
Count int64 `json:"count"`
|
||||
Last time.Time `json:"last"`
|
||||
}
|
||||
|
||||
// ReadRepo persists read buckets. Unlike the other repos this one is batch-
|
||||
// oriented: reads are telemetry and flushes carry many dirty buckets at once —
|
||||
// one file rewrite / one SQL transaction per flush, not per bucket.
|
||||
type ReadRepo interface {
|
||||
Load() ([]ReadStat, error)
|
||||
PutBatch(stats []ReadStat) error // upsert by (project,path,day,kind,actor)
|
||||
DeleteBatch(keys []ReadStatKey) error // used by retention fold
|
||||
}
|
||||
```
|
||||
|
||||
- `MetaStore` gains `Reads() ReadRepo`; file backend adds `reads.json`
|
||||
(same load-all / rewrite-atomically discipline, 0o755 dir mode — no secrets),
|
||||
SQL backend adds one table with PK `(project, path, day, kind, actor)` and
|
||||
the usual idempotent `CREATE TABLE IF NOT EXISTS` migration.
|
||||
- `db_conformance_test.go` gets ReadRepo cases like every other repo.
|
||||
|
||||
### Cardinality & retention
|
||||
|
||||
Worst realistic case (1k files × 30 actors × 400 days, every actor reading
|
||||
every file daily) is implausible; actual rows ≈ files-actually-read × active
|
||||
actors × active days. Retention keeps it bounded regardless:
|
||||
|
||||
- Config `reads.retention_days` (default **400** — enough for a year-over-year
|
||||
view). On the daily compaction pass (at load + once per day on the flush
|
||||
loop), buckets older than the horizon are **folded into the all-time row**
|
||||
`(project, path, "", kind, actor)` and deleted. All-time totals survive
|
||||
forever; per-day resolution ages out.
|
||||
|
||||
## Server plumbing
|
||||
|
||||
New `ReadLedger` service (`webapp/reads.go`), same shape as `DeviceRegistry`:
|
||||
in-memory state over a repo, with write throttling.
|
||||
|
||||
```go
|
||||
type ReadLedger struct {
|
||||
repo ReadRepo
|
||||
mu sync.Mutex
|
||||
byKey map[readKey]ReadStat // loaded at open, bumped in memory
|
||||
dirty map[readKey]struct{} // flushed every 30s and on Close
|
||||
seen map[visitKey]time.Time // debounce window
|
||||
}
|
||||
func (l *ReadLedger) Record(project, path, kind, actor string)
|
||||
func (l *ReadLedger) Heat(project, prefix string, since time.Time) map[string]HeatEntry
|
||||
func (l *ReadLedger) Close() error // final flush
|
||||
```
|
||||
|
||||
- `Server.Reads *ReadLedger` — nil means the feature is off (mirrors
|
||||
`Devices`/`Shares`). `Record` on a nil ledger is a no-op, so call sites
|
||||
stay unconditional.
|
||||
- **Recording sites**: `serveBlob` (covers `file` + `download`),
|
||||
`handleRender`, and `handleShared`. The `proj()` resolver already knows the
|
||||
project id; it stashes it in the request context so `recordRead(r, path,
|
||||
kind)` can pick it up without changing every handler signature.
|
||||
- **Never on the failure path**: record only after the path resolved and
|
||||
membership passed (`proj()` runs `projectAllowed` first — a 403 records
|
||||
nothing).
|
||||
- Flush errors degrade silently (log once), same "never break the request"
|
||||
posture as sync: telemetry must never 500 a page view.
|
||||
|
||||
## API
|
||||
|
||||
One endpoint, membership-gated like every per-project route:
|
||||
|
||||
```
|
||||
GET /api/p/<id>/heat?prefix=<folder/>&days=30
|
||||
→ { "since": "2026-06-11", "entries": {
|
||||
"wiki/onboarding.md": { "human": 42, "agent": 17, "share": 3,
|
||||
"readers": 6, "last_read": "2026-07-10T…" },
|
||||
… } }
|
||||
```
|
||||
|
||||
- `days=0` → all-time (daily buckets + the all-time fold).
|
||||
- `readers` = distinct human actors in the window. Actor identities never
|
||||
leave the server; this is the only trace of them.
|
||||
- **No writes-join on the server**: the frontend already has per-path last
|
||||
write time from `tree`, so the read×write matrix is a client-side join.
|
||||
One source of truth for write recency, no new endpoint.
|
||||
- `/api/config` gains `"reads": {"enabled": true|false}` so the frontend knows
|
||||
whether to fetch heat at all.
|
||||
|
||||
## Frontend
|
||||
|
||||
**Phase 1 — ambient heat (all members).**
|
||||
- Folder listing rows (`renderFolderListing`): a heat dot before the meta text,
|
||||
intensity 0–4 on a log scale of 30-day reads (any-kind), plus `· 42 reads`
|
||||
in `dl-meta`. Folders show the sum of their subtree.
|
||||
- File view meta line gains `· 42 reads/30d`.
|
||||
- One `heat?days=30` fetch per project, cached alongside the tree and
|
||||
refreshed with it.
|
||||
|
||||
**Phase 2 — Insights (admins + org owners).**
|
||||
- An "Insights" panel per project: SVG quadrant scatter, **x = days since last
|
||||
write (log), y = reads in window (log)**, one point per file. Quadrants:
|
||||
hot+fresh (healthy), **hot+stale (danger zone — fix these first)**,
|
||||
cold+fresh (new, unproven), cold+stale (archive candidates).
|
||||
- Below it, the danger-zone list ranked by `reads × staleness`, each row
|
||||
linking into the file with its history.
|
||||
- A human/agent toggle (or stacked dot colors) — "what do the agents live on"
|
||||
is a different question from "what do people open".
|
||||
- Dependency-free vanilla JS/SVG like the rest of the frontend.
|
||||
|
||||
## Agent reads (phase 3)
|
||||
|
||||
Agents read files from the *local synced folder* — the hub never sees those
|
||||
reads. The hooks pipeline shipped for session notes closes the gap:
|
||||
|
||||
1. **Hook**: each platform gets one more matcher — `Read` (Claude Code),
|
||||
`read_file|read_many_files` (Gemini), `read_file` (Hermes); Codex reads are
|
||||
mostly shell commands, so coverage there is best-effort. The hook runs
|
||||
`bdrive read-log`, which reads the hook JSON on stdin, extracts
|
||||
`tool_input.file_path`, and — iff the path is inside the mount and not
|
||||
ignored — appends `{path, time}` to a spool (`reads.jsonl`, `O_APPEND`,
|
||||
capped at ~10k lines) in the volume store. No network in the hook path.
|
||||
2. **Flush**: the sync cycle (daemon remote tick and one-shot `bdrive sync`)
|
||||
drains the spool best-effort to `POST /api/p/<id>/reads` — a new
|
||||
device-token-authenticated endpoint that records each entry as
|
||||
`kind=agent, actor=<device id>`. Offline → the spool just waits; a failed
|
||||
flush never fails the cycle. `remote/http.go` gains the client call as an
|
||||
optional capability interface (the `PutSigner` pattern) so `file://`
|
||||
test remotes simply don't report.
|
||||
3. Session attribution rides along free: the persisted session note is active
|
||||
during the same turn, so the hub can tag agent read batches with the same
|
||||
session id the writes carry (kept server-side; surfaced later if wanted).
|
||||
|
||||
## Privacy & config
|
||||
|
||||
```json
|
||||
"reads": { "enabled": true, "retention_days": 400 }
|
||||
```
|
||||
|
||||
- Default **on** (aggregate-only data; a hub admin can disable it).
|
||||
- Exposed data is only ever: per-path counts by kind, distinct-reader count,
|
||||
last-read time. Who-read-what is not queryable through any API, including
|
||||
admin APIs — the actor column exists solely to make `readers` honest.
|
||||
- Heat is member-visible (it helps everyone find the good docs); the Insights
|
||||
panel is admin/org-owner only, matching the saved intent.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit** (`reads_test.go`): debounce window, retention fold (daily → all-time),
|
||||
distinct-reader counting, nil-ledger no-ops, flush-on-close.
|
||||
- **Conformance**: ReadRepo ops across file/sqlite/postgres backends.
|
||||
- **Handler**: render/file/download/share record with the right kind; `/blob`
|
||||
and `/store/*` do not; non-member 403 records nothing; `heat` respects
|
||||
prefix + window; heat 404s in volume mode.
|
||||
- **Client** (phase 3): spool append from real hook JSON shapes (reuse the
|
||||
fixtures from `agenthooks_test.go`), flush drains + survives offline,
|
||||
cycle never fails on report errors.
|
||||
|
||||
## Phasing
|
||||
|
||||
1. **Ledger + heat** — ReadRepo (file/SQL), ReadLedger, recording sites,
|
||||
`/heat`, config block, folder heat dots + file read counts. Ships alone;
|
||||
human/share data starts accruing immediately.
|
||||
2. **Insights quadrant** — admin panel, danger-zone list. Pure frontend + the
|
||||
existing endpoints.
|
||||
3. **Agent reads** — `bdrive read-log`, hook matchers, spool, `POST /reads`,
|
||||
human/agent split in the UI. Depends on 1.
|
||||
|
||||
Phase 1 is the prerequisite for everything and is deliberately boring: one
|
||||
repo, one service, three record calls, one endpoint, two UI touches.
|
||||
|
||||
Phase 3 is the point of the feature, not tail work: human view counts are a
|
||||
commodity (every Confluence app has them); *agent* read visibility is the
|
||||
part nobody else can build. Phases are ordered by dependency, not value —
|
||||
ship 1 and 3 before polishing 2 if time is short.
|
||||
@@ -18,6 +18,7 @@ type MetaStore interface {
|
||||
Orgs() OrgRepo
|
||||
Shares() ShareRepo
|
||||
Devices() DeviceRepo
|
||||
Reads() ReadRepo
|
||||
Close() error
|
||||
}
|
||||
|
||||
@@ -56,3 +57,13 @@ type DeviceRepo interface {
|
||||
Load() ([]DeviceInfo, error)
|
||||
Put(d DeviceInfo) error
|
||||
}
|
||||
|
||||
// ReadRepo persists read-telemetry buckets (see ReadStat). Unlike the other
|
||||
// repos it is batch-oriented: reads are telemetry, and the ledger flushes many
|
||||
// dirty buckets at once — one file rewrite / one SQL transaction per flush,
|
||||
// not one write per bucket.
|
||||
type ReadRepo interface {
|
||||
Load() ([]ReadStat, error)
|
||||
PutBatch(stats []ReadStat) error // upsert by (project, path, day, kind, actor)
|
||||
DeleteBatch(keys []ReadStatKey) error
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func metaBackends(t *testing.T) []metaBackend {
|
||||
t.Fatalf("postgres reset: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.Exec(`DROP TABLE IF EXISTS accounts, tokens, auth_policy, projects, orgs, org_members, invites, shares, devices`)
|
||||
db.Exec(`DROP TABLE IF EXISTS accounts, tokens, auth_policy, projects, orgs, org_members, invites, shares, devices, read_stats`)
|
||||
},
|
||||
open: func(t *testing.T) MetaStore {
|
||||
s, err := OpenSQLStore("pgx", dsn)
|
||||
@@ -176,6 +176,17 @@ func TestMetaStoreConformance(t *testing.T) {
|
||||
}
|
||||
devices.Observe(DeviceInfo{ID: "d1", Name: "laptop", OS: "mac", User: "dev@x.io", IP: "1.2.3.4"})
|
||||
|
||||
reads, err := NewReadLedger(st.Reads(), 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reads.Record(p1.ID, "handbook.md", ReadKindHuman, "dev@x.io")
|
||||
reads.Record(p1.ID, "handbook.md", ReadKindHuman, "boss@x.io")
|
||||
reads.Record(p1.ID, "wiki/deep.md", ReadKindAgent, "d1")
|
||||
if err := reads.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -236,6 +247,21 @@ func TestMetaStoreConformance(t *testing.T) {
|
||||
if !ok || d.Name != "laptop" || d.IP != "1.2.3.4" {
|
||||
t.Fatalf("device lost across reload: %+v", d)
|
||||
}
|
||||
|
||||
reads2, err := NewReadLedger(st2.Reads(), 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heat := reads2.Heat(p1.ID, "", time.Time{})
|
||||
if e := heat["handbook.md"]; e.Human != 2 || e.Readers != 2 {
|
||||
t.Fatalf("read buckets lost across reload: %+v", e)
|
||||
}
|
||||
if e := heat["wiki/deep.md"]; e.Agent != 1 || e.Readers != 0 {
|
||||
t.Fatalf("agent read bucket lost across reload: %+v", e)
|
||||
}
|
||||
if sub := reads2.Heat(p1.ID, "wiki", time.Time{}); len(sub) != 1 {
|
||||
t.Fatalf("prefix heat = %+v, want only wiki/deep.md", sub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ type fileMetaStore struct {
|
||||
orgs *fileOrgRepo
|
||||
shares *fileShareRepo
|
||||
devices *fileDeviceRepo
|
||||
reads *fileReadRepo
|
||||
}
|
||||
|
||||
// OpenFileStore builds the file backend over dir, using the historical
|
||||
@@ -72,6 +73,7 @@ func OpenFileStore(dir string) (MetaStore, error) {
|
||||
orgs: newFileOrgRepo(filepath.Join(dir, "orgs.json")),
|
||||
shares: newFileShareRepo(filepath.Join(dir, "shares.json")),
|
||||
devices: newFileDeviceRepo(filepath.Join(dir, "devices.json")),
|
||||
reads: newFileReadRepo(filepath.Join(dir, "reads.json")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -80,6 +82,7 @@ func (s *fileMetaStore) Projects() ProjectRepo { return s.projects }
|
||||
func (s *fileMetaStore) Orgs() OrgRepo { return s.orgs }
|
||||
func (s *fileMetaStore) Shares() ShareRepo { return s.shares }
|
||||
func (s *fileMetaStore) Devices() DeviceRepo { return s.devices }
|
||||
func (s *fileMetaStore) Reads() ReadRepo { return s.reads }
|
||||
func (s *fileMetaStore) Close() error { return nil }
|
||||
|
||||
// ---- accounts (auth.json: users + tokens + policy) ----
|
||||
@@ -419,3 +422,72 @@ func (r *fileDeviceRepo) Put(d DeviceInfo) error {
|
||||
r.byID[d.ID] = d
|
||||
return r.write()
|
||||
}
|
||||
|
||||
// ---- reads (reads.json) ----
|
||||
|
||||
type fileReadRepo struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
byKey map[ReadStatKey]ReadStat
|
||||
}
|
||||
|
||||
func newFileReadRepo(path string) *fileReadRepo {
|
||||
return &fileReadRepo{path: path, byKey: map[ReadStatKey]ReadStat{}}
|
||||
}
|
||||
|
||||
func (r *fileReadRepo) Load() ([]ReadStat, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var f struct {
|
||||
Reads []ReadStat `json:"reads"`
|
||||
}
|
||||
if _, err := readJSONFile(r.path, &f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.byKey = map[ReadStatKey]ReadStat{}
|
||||
for _, st := range f.Reads {
|
||||
r.byKey[st.key()] = st
|
||||
}
|
||||
return f.Reads, nil
|
||||
}
|
||||
|
||||
func (r *fileReadRepo) write() error {
|
||||
var f struct {
|
||||
Reads []ReadStat `json:"reads"`
|
||||
}
|
||||
f.Reads = make([]ReadStat, 0, len(r.byKey))
|
||||
for _, st := range r.byKey {
|
||||
f.Reads = append(f.Reads, st)
|
||||
}
|
||||
sort.Slice(f.Reads, func(i, j int) bool {
|
||||
a, b := f.Reads[i], f.Reads[j]
|
||||
if a.Path != b.Path {
|
||||
return a.Path < b.Path
|
||||
}
|
||||
return a.Day < b.Day
|
||||
})
|
||||
data, err := json.Marshal(f) // telemetry: compact beats pretty
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 0700 dir: buckets carry actor emails, like auth.json carries accounts.
|
||||
return writeFileAtomic(r.path, append(data, '\n'), 0o700)
|
||||
}
|
||||
|
||||
func (r *fileReadRepo) PutBatch(stats []ReadStat) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, st := range stats {
|
||||
r.byKey[st.key()] = st
|
||||
}
|
||||
return r.write()
|
||||
}
|
||||
|
||||
func (r *fileReadRepo) DeleteBatch(keys []ReadStatKey) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, k := range keys {
|
||||
delete(r.byKey, k)
|
||||
}
|
||||
return r.write()
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ type sqlMetaStore struct {
|
||||
orgs *sqlOrgRepo
|
||||
shares *sqlShareRepo
|
||||
devices *sqlDeviceRepo
|
||||
reads *sqlReadRepo
|
||||
}
|
||||
|
||||
// OpenSQLStore opens (and migrates) a SQL metadata store. driver is "sqlite"
|
||||
@@ -66,6 +67,7 @@ func OpenSQLStore(driver, dsn string) (MetaStore, error) {
|
||||
s.orgs = &sqlOrgRepo{s}
|
||||
s.shares = &sqlShareRepo{s}
|
||||
s.devices = &sqlDeviceRepo{s}
|
||||
s.reads = &sqlReadRepo{s}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -74,6 +76,7 @@ func (s *sqlMetaStore) Projects() ProjectRepo { return s.projects }
|
||||
func (s *sqlMetaStore) Orgs() OrgRepo { return s.orgs }
|
||||
func (s *sqlMetaStore) Shares() ShareRepo { return s.shares }
|
||||
func (s *sqlMetaStore) Devices() DeviceRepo { return s.devices }
|
||||
func (s *sqlMetaStore) Reads() ReadRepo { return s.reads }
|
||||
func (s *sqlMetaStore) Close() error { return s.db.Close() }
|
||||
|
||||
// q rebinds ?-placeholders to $1,$2,… for Postgres; SQLite keeps ?.
|
||||
@@ -153,6 +156,11 @@ func (s *sqlMetaStore) migrate() error {
|
||||
`CREATE TABLE IF NOT EXISTS devices (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT '', os TEXT NOT NULL DEFAULT '',
|
||||
user_email TEXT NOT NULL DEFAULT '', ip TEXT NOT NULL DEFAULT '', last_seen TEXT NOT NULL DEFAULT '')`,
|
||||
`CREATE TABLE IF NOT EXISTS read_stats (
|
||||
project TEXT NOT NULL, path TEXT NOT NULL, day TEXT NOT NULL DEFAULT '',
|
||||
kind TEXT NOT NULL, actor TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0, last TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (project, path, day, kind, actor))`,
|
||||
}
|
||||
for _, st := range stmts {
|
||||
if _, err := s.db.Exec(st); err != nil {
|
||||
@@ -462,3 +470,59 @@ func (r *sqlDeviceRepo) Put(d DeviceInfo) error {
|
||||
ip=excluded.ip, last_seen=excluded.last_seen`,
|
||||
d.ID, d.Name, d.OS, d.User, d.IP, tenc(d.LastSeen))
|
||||
}
|
||||
|
||||
// ---- reads ----
|
||||
|
||||
type sqlReadRepo struct{ s *sqlMetaStore }
|
||||
|
||||
func (r *sqlReadRepo) Load() ([]ReadStat, error) {
|
||||
rows, err := r.s.db.Query(`SELECT project, path, day, kind, actor, count, last FROM read_stats`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ReadStat
|
||||
for rows.Next() {
|
||||
var st ReadStat
|
||||
var last string
|
||||
if err := rows.Scan(&st.Project, &st.Path, &st.Day, &st.Kind, &st.Actor, &st.Count, &last); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st.Last = tdec(last)
|
||||
out = append(out, st)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *sqlReadRepo) PutBatch(stats []ReadStat) error {
|
||||
tx, err := r.s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, st := range stats {
|
||||
if _, err := tx.Exec(r.s.q(`INSERT INTO read_stats (project,path,day,kind,actor,count,last)
|
||||
VALUES (?,?,?,?,?,?,?)
|
||||
ON CONFLICT(project,path,day,kind,actor) DO UPDATE SET count=excluded.count, last=excluded.last`),
|
||||
st.Project, st.Path, st.Day, st.Kind, st.Actor, st.Count, tenc(st.Last)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *sqlReadRepo) DeleteBatch(keys []ReadStatKey) error {
|
||||
tx, err := r.s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, k := range keys {
|
||||
if _, err := tx.Exec(r.s.q(`DELETE FROM read_stats
|
||||
WHERE project = ? AND path = ? AND day = ? AND kind = ? AND actor = ?`),
|
||||
k.Project, k.Path, k.Day, k.Kind, k.Actor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
package webapp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Read telemetry: who consumes what, aggregated. Together with the write
|
||||
// history the journals already carry, this completes the read×write matrix —
|
||||
// heavily-read + long-unwritten is the danger zone an admin should fix first
|
||||
// (see docs/design/read-heatmap.md).
|
||||
//
|
||||
// What counts as a read: viewer file/render/download hits (human), share-link
|
||||
// hits (share), and agent tool reads reported by syncing devices (agent).
|
||||
// /store/* sync traffic is replication, not reading, and is never counted;
|
||||
// history /blob views are spelunking, not consumption, and aren't either.
|
||||
//
|
||||
// Privacy: rows are daily aggregation buckets, never an event log. The actor
|
||||
// column (account email / device id / share token) exists only to count
|
||||
// distinct readers and never appears in an API response.
|
||||
|
||||
// Read kinds.
|
||||
const (
|
||||
ReadKindHuman = "human"
|
||||
ReadKindAgent = "agent"
|
||||
ReadKindShare = "share"
|
||||
)
|
||||
|
||||
// ReadStat is one aggregation bucket: reads of one path by one actor on one
|
||||
// day. Day == "" is the all-time fold that survives retention.
|
||||
type ReadStat struct {
|
||||
Project string `json:"project"`
|
||||
Path string `json:"path"`
|
||||
Day string `json:"day"` // "2006-01-02" UTC, or "" for all-time
|
||||
Kind string `json:"kind"`
|
||||
Actor string `json:"actor"`
|
||||
Count int64 `json:"count"`
|
||||
Last time.Time `json:"last"`
|
||||
}
|
||||
|
||||
// ReadStatKey identifies one bucket.
|
||||
type ReadStatKey struct {
|
||||
Project, Path, Day, Kind, Actor string
|
||||
}
|
||||
|
||||
func (s ReadStat) key() ReadStatKey {
|
||||
return ReadStatKey{s.Project, s.Path, s.Day, s.Kind, s.Actor}
|
||||
}
|
||||
|
||||
// HeatEntry is the per-path aggregate the heat API returns. Counts only —
|
||||
// never identities.
|
||||
type HeatEntry struct {
|
||||
Human int64 `json:"human,omitempty"`
|
||||
Agent int64 `json:"agent,omitempty"`
|
||||
Share int64 `json:"share,omitempty"`
|
||||
Readers int `json:"readers,omitempty"` // distinct human readers
|
||||
LastRead time.Time `json:"last_read,omitzero"`
|
||||
}
|
||||
|
||||
const (
|
||||
// readDebounce collapses request storms (reloads, render-then-raw double
|
||||
// fetches) into visits: repeat reads of a path by the same actor within
|
||||
// the window don't count again.
|
||||
readDebounce = 10 * time.Minute
|
||||
// readFlushEvery throttles persistence; dirty buckets ride in memory
|
||||
// between flushes, so a crash loses at most this much telemetry.
|
||||
readFlushEvery = 30 * time.Second
|
||||
// DefaultReadRetentionDays is how long daily buckets keep per-day
|
||||
// resolution before folding into the all-time row.
|
||||
DefaultReadRetentionDays = 400
|
||||
)
|
||||
|
||||
// ReadLedger is the in-memory read-telemetry service over a ReadRepo, in the
|
||||
// mold of DeviceRegistry: reads stay in memory, writes are throttled. There is
|
||||
// no background goroutine — flushes piggyback on Record calls, and telemetry
|
||||
// failures never surface to the request that triggered them.
|
||||
type ReadLedger struct {
|
||||
repo ReadRepo
|
||||
retention time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
byKey map[ReadStatKey]ReadStat
|
||||
dirty map[ReadStatKey]bool
|
||||
pendingDel []ReadStatKey // retention deletions awaiting a successful flush
|
||||
seen map[ReadStatKey]time.Time // debounce; Day field unused ("")
|
||||
lastFlush time.Time
|
||||
warned bool
|
||||
}
|
||||
|
||||
// NewReadLedger loads the ledger and immediately folds buckets older than the
|
||||
// retention horizon into their all-time rows. retentionDays <= 0 means the
|
||||
// default.
|
||||
func NewReadLedger(repo ReadRepo, retentionDays int) (*ReadLedger, error) {
|
||||
if retentionDays <= 0 {
|
||||
retentionDays = DefaultReadRetentionDays
|
||||
}
|
||||
l := &ReadLedger{
|
||||
repo: repo,
|
||||
retention: time.Duration(retentionDays) * 24 * time.Hour,
|
||||
byKey: map[ReadStatKey]ReadStat{},
|
||||
dirty: map[ReadStatKey]bool{},
|
||||
seen: map[ReadStatKey]time.Time{},
|
||||
lastFlush: time.Now(),
|
||||
}
|
||||
stats, err := repo.Load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, st := range stats {
|
||||
l.byKey[st.key()] = st
|
||||
}
|
||||
// Fold anything past the retention horizon right away. A failed persist
|
||||
// is not a boot failure — the fold stays dirty and later flushes retry.
|
||||
l.mu.Lock()
|
||||
l.compactLocked()
|
||||
if err := l.persistLocked(); err != nil {
|
||||
log.Printf("beardrive: read telemetry compact failed (will retry): %v", err)
|
||||
}
|
||||
l.mu.Unlock()
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// OpenReadLedger loads the file-backed ledger at path.
|
||||
func OpenReadLedger(path string, retentionDays int) (*ReadLedger, error) {
|
||||
return NewReadLedger(newFileReadRepo(path), retentionDays)
|
||||
}
|
||||
|
||||
// Record counts one read. Nil-safe and never fails: telemetry must not break
|
||||
// the page view (or sync cycle) that triggered it.
|
||||
func (l *ReadLedger) Record(project, path, kind, actor string) {
|
||||
if l == nil || project == "" || path == "" {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
visit := ReadStatKey{Project: project, Path: path, Kind: kind, Actor: actor}
|
||||
if t, ok := l.seen[visit]; ok && now.Sub(t) < readDebounce {
|
||||
return
|
||||
}
|
||||
l.seen[visit] = now
|
||||
key := visit
|
||||
key.Day = now.Format("2006-01-02")
|
||||
st := l.byKey[key]
|
||||
st.Project, st.Path, st.Day, st.Kind, st.Actor = project, path, key.Day, kind, actor
|
||||
st.Count++
|
||||
st.Last = now
|
||||
l.byKey[key] = st
|
||||
l.dirty[key] = true
|
||||
if now.Sub(l.lastFlush) >= readFlushEvery {
|
||||
l.flushLocked()
|
||||
}
|
||||
}
|
||||
|
||||
// Heat aggregates reads per path for one project. since bounds the window
|
||||
// (zero = all time, including retention folds); prefix "" means the whole
|
||||
// project, otherwise paths under "<prefix>/".
|
||||
func (l *ReadLedger) Heat(project, prefix string, since time.Time) map[string]HeatEntry {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
sinceDay := ""
|
||||
if !since.IsZero() {
|
||||
sinceDay = since.UTC().Format("2006-01-02")
|
||||
}
|
||||
prefix = strings.TrimSuffix(prefix, "/")
|
||||
out := map[string]HeatEntry{}
|
||||
humans := map[string]map[string]bool{} // path → distinct human actors
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for key, st := range l.byKey {
|
||||
if key.Project != project {
|
||||
continue
|
||||
}
|
||||
if prefix != "" && !strings.HasPrefix(key.Path, prefix+"/") {
|
||||
continue
|
||||
}
|
||||
if key.Day == "" {
|
||||
if sinceDay != "" {
|
||||
continue // all-time fold is older than any windowed query
|
||||
}
|
||||
} else if key.Day < sinceDay {
|
||||
continue
|
||||
}
|
||||
e := out[key.Path]
|
||||
switch key.Kind {
|
||||
case ReadKindHuman:
|
||||
e.Human += st.Count
|
||||
set := humans[key.Path]
|
||||
if set == nil {
|
||||
set = map[string]bool{}
|
||||
humans[key.Path] = set
|
||||
}
|
||||
set[key.Actor] = true
|
||||
case ReadKindAgent:
|
||||
e.Agent += st.Count
|
||||
case ReadKindShare:
|
||||
e.Share += st.Count
|
||||
}
|
||||
if st.Last.After(e.LastRead) {
|
||||
e.LastRead = st.Last
|
||||
}
|
||||
out[key.Path] = e
|
||||
}
|
||||
for p, set := range humans {
|
||||
e := out[p]
|
||||
e.Readers = len(set)
|
||||
out[p] = e
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Close flushes any pending buckets.
|
||||
func (l *ReadLedger) Close() error {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.flushLocked()
|
||||
if n := len(l.dirty); n > 0 {
|
||||
return fmt.Errorf("read ledger: flush failed, %d buckets pending", n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// flushLocked persists dirty buckets (and, once a day, retention folds),
|
||||
// pruning the debounce map along the way. Failures keep the buckets dirty for
|
||||
// the next attempt and log once — telemetry never breaks a request.
|
||||
func (l *ReadLedger) flushLocked() {
|
||||
now := time.Now()
|
||||
l.lastFlush = now
|
||||
for k, t := range l.seen {
|
||||
if now.Sub(t) >= readDebounce {
|
||||
delete(l.seen, k)
|
||||
}
|
||||
}
|
||||
l.compactLocked()
|
||||
if err := l.persistLocked(); err != nil {
|
||||
if !l.warned {
|
||||
l.warned = true
|
||||
log.Printf("beardrive: read telemetry flush failed (will retry): %v", err)
|
||||
}
|
||||
} else {
|
||||
l.warned = false
|
||||
}
|
||||
}
|
||||
|
||||
// compactLocked folds daily buckets older than the retention horizon into
|
||||
// their all-time rows, queueing the daily rows for deletion. Callers hold mu.
|
||||
func (l *ReadLedger) compactLocked() {
|
||||
horizon := time.Now().UTC().Add(-l.retention).Format("2006-01-02")
|
||||
for key, st := range l.byKey {
|
||||
if key.Day == "" || key.Day >= horizon {
|
||||
continue
|
||||
}
|
||||
fold := key
|
||||
fold.Day = ""
|
||||
agg := l.byKey[fold]
|
||||
agg.Project, agg.Path, agg.Kind, agg.Actor = st.Project, st.Path, st.Kind, st.Actor
|
||||
agg.Day = ""
|
||||
agg.Count += st.Count
|
||||
if st.Last.After(agg.Last) {
|
||||
agg.Last = st.Last
|
||||
}
|
||||
l.byKey[fold] = agg
|
||||
l.dirty[fold] = true
|
||||
delete(l.byKey, key)
|
||||
delete(l.dirty, key)
|
||||
l.pendingDel = append(l.pendingDel, key)
|
||||
}
|
||||
}
|
||||
|
||||
// persistLocked writes queued deletions and dirty buckets through the repo.
|
||||
// Callers hold mu. Both queues survive a failure so the next flush retries —
|
||||
// dropping a deletion would resurrect folded rows on the next load and
|
||||
// double-count them.
|
||||
func (l *ReadLedger) persistLocked() error {
|
||||
if len(l.pendingDel) > 0 {
|
||||
if err := l.repo.DeleteBatch(l.pendingDel); err != nil {
|
||||
return err
|
||||
}
|
||||
l.pendingDel = nil
|
||||
}
|
||||
if len(l.dirty) == 0 {
|
||||
return nil
|
||||
}
|
||||
batch := make([]ReadStat, 0, len(l.dirty))
|
||||
for key := range l.dirty {
|
||||
batch = append(batch, l.byKey[key])
|
||||
}
|
||||
if err := l.repo.PutBatch(batch); err != nil {
|
||||
return err
|
||||
}
|
||||
l.dirty = map[ReadStatKey]bool{}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- server integration ----
|
||||
|
||||
// ctxProjectKey carries the resolved project id from the proj() route
|
||||
// resolver to handlers that record reads.
|
||||
type ctxProjectKey struct{}
|
||||
|
||||
func withProjectID(r *http.Request, id string) *http.Request {
|
||||
return r.WithContext(context.WithValue(r.Context(), ctxProjectKey{}, id))
|
||||
}
|
||||
|
||||
func projectID(r *http.Request) string {
|
||||
id, _ := r.Context().Value(ctxProjectKey{}).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
// recordRead counts a human read of path for the request's project. No-op
|
||||
// outside hub mode (no project id) or when read tracking is off.
|
||||
func (s *Server) recordRead(r *http.Request, path string) {
|
||||
if s.Reads == nil {
|
||||
return
|
||||
}
|
||||
project := projectID(r)
|
||||
if project == "" {
|
||||
return
|
||||
}
|
||||
actor := s.requestUser(r).Email
|
||||
if actor == "" {
|
||||
actor = "anonymous"
|
||||
}
|
||||
s.Reads.Record(project, path, ReadKindHuman, actor)
|
||||
}
|
||||
|
||||
// handleHeat serves per-path read aggregates: ?prefix= bounds to a folder,
|
||||
// ?days= bounds the window (default 30, 0 = all time). Counts only — actor
|
||||
// identities never leave the server.
|
||||
func (s *Server) handleHeat(v *volume, w http.ResponseWriter, r *http.Request) {
|
||||
if s.Reads == nil {
|
||||
http.Error(w, "read tracking is not enabled on this server", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = v
|
||||
q := r.URL.Query()
|
||||
days := 30
|
||||
if raw := q.Get("days"); raw != "" {
|
||||
var err error
|
||||
if days, err = strconv.Atoi(raw); err != nil || days < 0 {
|
||||
http.Error(w, "invalid days", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
var since time.Time
|
||||
if days > 0 {
|
||||
since = time.Now().UTC().AddDate(0, 0, -days)
|
||||
}
|
||||
entries := s.Reads.Heat(projectID(r), q.Get("prefix"), since)
|
||||
out := map[string]any{"entries": entries}
|
||||
if !since.IsZero() {
|
||||
out["since"] = since.Format("2006-01-02")
|
||||
}
|
||||
writeJSON(w, out)
|
||||
}
|
||||
|
||||
// handleReadReport ingests agent reads from a syncing device: the client's
|
||||
// read spool, drained best-effort at sync time. Requires a device identity —
|
||||
// the device id is the actor, so reads count as agent traffic.
|
||||
func (s *Server) handleReadReport(v *volume, w http.ResponseWriter, r *http.Request) {
|
||||
if s.Reads == nil {
|
||||
http.Error(w, "read tracking is not enabled on this server", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = v
|
||||
device := r.Header.Get("X-Bdrive-Device")
|
||||
if device == "" {
|
||||
http.Error(w, "agent read reports need a device identity", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Reads []struct {
|
||||
Path string `json:"path"`
|
||||
// Time is accepted for forward compatibility but buckets use
|
||||
// server time: client clocks are unreliable and late flushes are
|
||||
// telemetry noise, not data loss.
|
||||
Time time.Time `json:"time,omitzero"`
|
||||
} `json:"reads"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(req.Reads) > 4096 {
|
||||
http.Error(w, "too many reads in one report", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
s.observeDevice(r)
|
||||
project := projectID(r)
|
||||
n := 0
|
||||
for _, e := range req.Reads {
|
||||
if e.Path == "" || strings.Contains(e.Path, "..") {
|
||||
continue
|
||||
}
|
||||
s.Reads.Record(project, e.Path, ReadKindAgent, device)
|
||||
n++
|
||||
}
|
||||
writeJSON(w, map[string]any{"accepted": n})
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package webapp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// doHdr is do() with request headers (device identity on read reports).
|
||||
func doHdr(t *testing.T, h http.Handler, method, url string, body any, hdr map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(method, url, bytes.NewReader(data))
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func openTestLedger(t *testing.T, retentionDays int) (*ReadLedger, *fileReadRepo) {
|
||||
t.Helper()
|
||||
repo := newFileReadRepo(filepath.Join(t.TempDir(), "reads.json"))
|
||||
l, err := NewReadLedger(repo, retentionDays)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return l, repo
|
||||
}
|
||||
|
||||
func TestReadLedgerDebounce(t *testing.T) {
|
||||
l, _ := openTestLedger(t, 0)
|
||||
// A reload storm and the render-then-raw double fetch are one visit…
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "alice@x.io")
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "alice@x.io")
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "alice@x.io")
|
||||
// …but a different actor, kind, or path counts on its own.
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "bob@x.io")
|
||||
l.Record("p-1", "a.md", ReadKindAgent, "alice@x.io")
|
||||
l.Record("p-1", "b.md", ReadKindHuman, "alice@x.io")
|
||||
heat := l.Heat("p-1", "", time.Time{})
|
||||
if e := heat["a.md"]; e.Human != 2 || e.Agent != 1 || e.Readers != 2 {
|
||||
t.Fatalf("a.md = %+v, want human 2, agent 1, readers 2", e)
|
||||
}
|
||||
if e := heat["b.md"]; e.Human != 1 || e.Readers != 1 {
|
||||
t.Fatalf("b.md = %+v", e)
|
||||
}
|
||||
if heat["a.md"].LastRead.IsZero() {
|
||||
t.Fatal("last_read not set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLedgerWindow(t *testing.T) {
|
||||
l, repo := openTestLedger(t, 0)
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "alice@x.io")
|
||||
if err := l.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An old bucket inside retention: counted all-time, outside a 7-day window.
|
||||
old := ReadStat{Project: "p-1", Path: "a.md", Day: "2026-01-01", Kind: ReadKindHuman,
|
||||
Actor: "carol@x.io", Count: 5, Last: time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)}
|
||||
if err := repo.PutBatch([]ReadStat{old}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l2, err := NewReadLedger(repo, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := l2.Heat("p-1", "", time.Time{})["a.md"]; e.Human != 6 || e.Readers != 2 {
|
||||
t.Fatalf("all-time = %+v, want human 6, readers 2", e)
|
||||
}
|
||||
week := time.Now().UTC().AddDate(0, 0, -7)
|
||||
if e := l2.Heat("p-1", "", week)["a.md"]; e.Human != 1 || e.Readers != 1 {
|
||||
t.Fatalf("windowed = %+v, want only today's read", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLedgerRetentionFold(t *testing.T) {
|
||||
repo := newFileReadRepo(filepath.Join(t.TempDir(), "reads.json"))
|
||||
seed := []ReadStat{
|
||||
{Project: "p-1", Path: "a.md", Day: "2020-01-01", Kind: ReadKindHuman, Actor: "alice@x.io", Count: 3,
|
||||
Last: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{Project: "p-1", Path: "a.md", Day: "2020-01-02", Kind: ReadKindHuman, Actor: "alice@x.io", Count: 2,
|
||||
Last: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC)},
|
||||
{Project: "p-1", Path: "a.md", Day: time.Now().UTC().Format("2006-01-02"), Kind: ReadKindHuman,
|
||||
Actor: "alice@x.io", Count: 1, Last: time.Now().UTC()},
|
||||
}
|
||||
if err := repo.PutBatch(seed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, err := NewReadLedger(repo, 30)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// All-time totals survive the fold; per-day resolution ages out.
|
||||
if e := l.Heat("p-1", "", time.Time{})["a.md"]; e.Human != 6 || e.Readers != 1 {
|
||||
t.Fatalf("after fold = %+v, want human 6, readers 1", e)
|
||||
}
|
||||
if err := l.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := repo.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var folds, dailies int
|
||||
for _, st := range rows {
|
||||
if st.Day == "" {
|
||||
folds++
|
||||
if st.Count != 5 {
|
||||
t.Fatalf("fold count = %d, want 5", st.Count)
|
||||
}
|
||||
} else {
|
||||
dailies++
|
||||
}
|
||||
}
|
||||
if folds != 1 || dailies != 1 {
|
||||
t.Fatalf("rows after fold: %d folds, %d dailies; want 1 and 1 (%+v)", folds, dailies, rows)
|
||||
}
|
||||
// Reloading must not double-count: the fold replaced the old rows.
|
||||
l2, err := NewReadLedger(repo, 30)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := l2.Heat("p-1", "", time.Time{})["a.md"]; e.Human != 6 {
|
||||
t.Fatalf("after reload = %+v, want human still 6", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLedgerNil(t *testing.T) {
|
||||
var l *ReadLedger
|
||||
l.Record("p-1", "a.md", ReadKindHuman, "x") // must not panic
|
||||
if l.Heat("p-1", "", time.Time{}) != nil {
|
||||
t.Fatal("nil ledger heat should be nil")
|
||||
}
|
||||
if err := l.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeatAPI drives the recording sites and the heat/report endpoints
|
||||
// through the real handler: renders and downloads count (debounced), the
|
||||
// store sync proxy and history blob views never count, and device-reported
|
||||
// agent reads land as agent traffic.
|
||||
func TestHeatAPI(t *testing.T) {
|
||||
srv, p, root := newHub(t, false, nil)
|
||||
f := newFakeRemoteAt(t, filepath.Join(root, p.ID))
|
||||
f.putAs("dev1", "alice@x.io", "Alice", "wiki/plan.md", "# plan")
|
||||
f.putAs("dev1", "alice@x.io", "Alice", "top.md", "top")
|
||||
var err error
|
||||
srv.Reads, err = OpenReadLedger(filepath.Join(t.TempDir(), "reads.json"), 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := srv.Handler()
|
||||
base := "/api/p/" + p.ID + "/"
|
||||
|
||||
// render + raw fetch of the same file = one visit (debounced)
|
||||
for _, u := range []string{base + "render?path=wiki/plan.md", base + "file?path=wiki/plan.md"} {
|
||||
if rec := do(t, h, "GET", u, nil); rec.Code != 200 {
|
||||
t.Fatalf("GET %s: %d %s", u, rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
// store proxy traffic is replication, not reading
|
||||
do(t, h, "GET", base+"store/list?prefix=journal/", nil)
|
||||
do(t, h, "GET", base+"store/object?key=journal/dev1.jsonl", nil)
|
||||
|
||||
rec := do(t, h, "GET", base+"heat", nil)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("heat: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var out struct {
|
||||
Entries map[string]HeatEntry `json:"entries"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := out.Entries["wiki/plan.md"]; e.Human != 1 || e.Readers != 1 {
|
||||
t.Fatalf("plan.md heat = %+v, want one human visit", e)
|
||||
}
|
||||
if len(out.Entries) != 1 || out.Since == "" {
|
||||
t.Fatalf("heat = %+v; store traffic must not count", out)
|
||||
}
|
||||
|
||||
// an agent device reports its local tool reads
|
||||
report := map[string]any{
|
||||
"reads": []map[string]string{{"path": "wiki/plan.md"}, {"path": "top.md"}, {"path": "../evil"}},
|
||||
}
|
||||
rec = doHdr(t, h, "POST", base+"reads", report, map[string]string{"X-Bdrive-Device": "dev1"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("report: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var acc struct {
|
||||
Accepted int `json:"accepted"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &acc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if acc.Accepted != 2 {
|
||||
t.Fatalf("accepted = %d, want 2 (traversal path dropped)", acc.Accepted)
|
||||
}
|
||||
// …and without a device identity the report is rejected
|
||||
if rec := doHdr(t, h, "POST", base+"reads", report, nil); rec.Code != 400 {
|
||||
t.Fatalf("device-less report: %d, want 400", rec.Code)
|
||||
}
|
||||
|
||||
rec = do(t, h, "GET", base+"heat?prefix=wiki", nil)
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := out.Entries["wiki/plan.md"]; e.Human != 1 || e.Agent != 1 {
|
||||
t.Fatalf("plan.md after report = %+v, want human 1 + agent 1", e)
|
||||
}
|
||||
if _, ok := out.Entries["top.md"]; ok {
|
||||
t.Fatal("prefix filter leaked top.md")
|
||||
}
|
||||
|
||||
if rec := do(t, h, "GET", base+"heat?days=x", nil); rec.Code != 400 {
|
||||
t.Fatalf("bad days: %d, want 400", rec.Code)
|
||||
}
|
||||
|
||||
// a hub without read tracking 404s cleanly
|
||||
srv.Reads = nil
|
||||
if rec := do(t, h, "GET", base+"heat", nil); rec.Code != 404 {
|
||||
t.Fatalf("disabled heat: %d, want 404", rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,9 @@ type Server struct {
|
||||
Devices *DeviceRegistry
|
||||
// Shares, when set, enables public share links (/s/<token>).
|
||||
Shares *ShareDB
|
||||
// Reads, when set, aggregates read telemetry (viewer, share, and agent
|
||||
// reads) for the heat API. Nil means read tracking is off.
|
||||
Reads *ReadLedger
|
||||
// Orgs, when set, walls projects off by organization membership.
|
||||
Orgs *OrgDB
|
||||
// Quota, when set, enforces plan limits (managed deployments). Nil
|
||||
@@ -305,7 +308,9 @@ func (s *Server) Handler() http.Handler {
|
||||
http.Error(w, "you are not a member of this project's organization", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
h(v, w, r)
|
||||
// Read recording (and anything else downstream) finds the project
|
||||
// id in the context; membership has already passed at this point.
|
||||
h(v, w, withProjectID(r, id))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,6 +353,8 @@ func (s *Server) Handler() http.Handler {
|
||||
|
||||
mux.HandleFunc("GET /api/p/{project}/history", proj(s.handleHistory))
|
||||
mux.HandleFunc("GET /api/p/{project}/blob", proj(s.handleBlob))
|
||||
mux.HandleFunc("GET /api/p/{project}/heat", proj(s.handleHeat))
|
||||
mux.HandleFunc("POST /api/p/{project}/reads", proj(s.handleReadReport))
|
||||
mux.HandleFunc("POST /api/p/{project}/shares", proj(s.handleShareCreate))
|
||||
mux.HandleFunc("GET /api/p/{project}/shares", proj(s.handleShareList))
|
||||
mux.HandleFunc("DELETE /api/shares/{token}", s.handleShareRevoke)
|
||||
@@ -429,7 +436,8 @@ func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
|
||||
"upload": map[string]any{
|
||||
"enabled": s.Upload.Enabled,
|
||||
},
|
||||
"auth": auth,
|
||||
"auth": auth,
|
||||
"reads": map[string]any{"enabled": s.Reads != nil},
|
||||
}
|
||||
if me.Email != "" {
|
||||
out["me"] = map[string]string{"email": me.Email, "name": me.Name}
|
||||
@@ -607,12 +615,15 @@ func lookup(v *volume, r *http.Request) (string, FileInfo, int, error) {
|
||||
return p, fi, 0, nil
|
||||
}
|
||||
|
||||
func serveBlob(v *volume, w http.ResponseWriter, r *http.Request, attach bool) {
|
||||
func (s *Server) serveBlob(v *volume, w http.ResponseWriter, r *http.Request, attach bool) {
|
||||
p, fi, code, err := lookup(v, r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), code)
|
||||
return
|
||||
}
|
||||
// Count the read before the ETag check: a 304 render is still a person
|
||||
// reading the file, and skipping it would undercount the hottest pages.
|
||||
s.recordRead(r, p)
|
||||
etag := `"` + fi.Blob + `"`
|
||||
if r.Header.Get("If-None-Match") == etag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
@@ -634,11 +645,11 @@ func serveBlob(v *volume, w http.ResponseWriter, r *http.Request, attach bool) {
|
||||
}
|
||||
|
||||
func (s *Server) handleFile(v *volume, w http.ResponseWriter, r *http.Request) {
|
||||
serveBlob(v, w, r, false)
|
||||
s.serveBlob(v, w, r, false)
|
||||
}
|
||||
|
||||
func (s *Server) handleDownload(v *volume, w http.ResponseWriter, r *http.Request) {
|
||||
serveBlob(v, w, r, true)
|
||||
s.serveBlob(v, w, r, true)
|
||||
}
|
||||
|
||||
func (s *Server) handleRender(v *volume, w http.ResponseWriter, r *http.Request) {
|
||||
@@ -647,6 +658,7 @@ func (s *Server) handleRender(v *volume, w http.ResponseWriter, r *http.Request)
|
||||
http.Error(w, err.Error(), code)
|
||||
return
|
||||
}
|
||||
s.recordRead(r, p)
|
||||
rc, err := v.source.Open(r.Context(), p, fi)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("fetch content: %v", err), http.StatusBadGateway)
|
||||
|
||||
@@ -243,6 +243,9 @@ func (s *Server) handleShared(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "the shared file no longer exists", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// A share hit is external consumption. Actor is token+IP: one audience
|
||||
// member reloading is debounced to a visit, distinct visitors still count.
|
||||
s.Reads.Record(sh.Project, sh.Path, ReadKindShare, sh.Token+"/"+clientIP(r))
|
||||
|
||||
// Sandbox everything under /s/: shared content executes in an opaque
|
||||
// origin (scripts allowed — charts in reports — but no cookies, no
|
||||
|
||||
@@ -21,6 +21,8 @@ let currentProject = null; // hub mode: the selected project
|
||||
let apiBase = "/api/"; // volume-scoped endpoint prefix
|
||||
let orgs = []; // hub mode: the orgs this account belongs to
|
||||
let joinedOrgId = null; // org just joined via an invite this page-load
|
||||
let heatMap = null; // hub: path → 30-day read counts, from the heat API
|
||||
let heatAt = 0; // when heatMap was last fetched (ms)
|
||||
|
||||
const fileURL = (p) => apiBase + "file?path=" + encodeURIComponent(p);
|
||||
|
||||
@@ -148,6 +150,8 @@ function selectProject(p, path) {
|
||||
currentProject = p;
|
||||
expanded = new Set(); // fresh collapse state for the new project's tree
|
||||
treeFirstLoad = true;
|
||||
heatMap = null; // the old project's heat means nothing here
|
||||
heatAt = 0;
|
||||
apiBase = "/api/p/" + p.id + "/";
|
||||
$("vault-name").textContent = p.name;
|
||||
document.title = p.name + " — BearDrive";
|
||||
@@ -694,6 +698,60 @@ async function refreshTree() {
|
||||
if (currentPath && dirIndex.has(currentPath) && $("content").querySelector(".dirlist")) {
|
||||
renderFolderListing(currentPath);
|
||||
}
|
||||
refreshHeat();
|
||||
}
|
||||
|
||||
/* ---- read heat ----
|
||||
30-day read counts per path from the heat API (hub only). Counts only —
|
||||
the server never says who read what. Fetched lazily alongside the tree. */
|
||||
async function refreshHeat(force) {
|
||||
if (!(serverConfig.mode === "hub" && currentProject)) return;
|
||||
if (!(serverConfig.reads && serverConfig.reads.enabled)) return;
|
||||
if (!force && Date.now() - heatAt < 60000) return;
|
||||
heatAt = Date.now(); // set before the fetch so failures don't hammer
|
||||
let out;
|
||||
try {
|
||||
out = await getJSON(apiBase + "heat?days=30");
|
||||
} catch { return; } // keep the last good heat
|
||||
heatMap = out.entries || {};
|
||||
if (currentPath && dirIndex.has(currentPath) && $("content").querySelector(".dirlist")) {
|
||||
renderFolderListing(currentPath);
|
||||
}
|
||||
}
|
||||
|
||||
/* Heat for one listing entry: a file's own bucket, or the subtree sum for a
|
||||
folder. Null when there is nothing to show. */
|
||||
function heatFor(path, isDir) {
|
||||
if (!heatMap) return null;
|
||||
if (!isDir) return heatMap[path] || null;
|
||||
const agg = { human: 0, agent: 0, share: 0 };
|
||||
for (const [p, e] of Object.entries(heatMap)) {
|
||||
if (!p.startsWith(path + "/")) continue;
|
||||
agg.human += e.human || 0;
|
||||
agg.agent += e.agent || 0;
|
||||
agg.share += e.share || 0;
|
||||
}
|
||||
return agg.human || agg.agent || agg.share ? agg : null;
|
||||
}
|
||||
|
||||
function heatTotal(e) { return (e.human || 0) + (e.agent || 0) + (e.share || 0); }
|
||||
|
||||
function heatText(e) {
|
||||
const total = heatTotal(e);
|
||||
if (!total) return "";
|
||||
let s = total + (total === 1 ? " read" : " reads");
|
||||
if (e.agent) s += " (" + e.agent + " agent)";
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Dot intensity 1–4, log-ish steps: 1–2, 3–9, 10–29, 30+ reads. */
|
||||
function heatLevel(e) {
|
||||
const total = heatTotal(e);
|
||||
if (!total) return 0;
|
||||
if (total < 3) return 1;
|
||||
if (total < 10) return 2;
|
||||
if (total < 30) return 3;
|
||||
return 4;
|
||||
}
|
||||
|
||||
function renderChildren(children) {
|
||||
@@ -865,6 +923,8 @@ function renderFolderListing(p) {
|
||||
const counts = [];
|
||||
if (dirs) counts.push(dirs + (dirs === 1 ? " folder" : " folders"));
|
||||
if (files) counts.push(files + (files === 1 ? " file" : " files"));
|
||||
const folderHeat = heatFor(p, true);
|
||||
if (folderHeat) counts.push(heatText(folderHeat) + " in 30 days");
|
||||
el(wrap, "p", "dl-sub", counts.join(" · ") || "Empty folder");
|
||||
if (!kids.length) {
|
||||
el(wrap, "div", "dl-empty", "Nothing in this folder yet.");
|
||||
@@ -888,6 +948,12 @@ function renderFolderListing(p) {
|
||||
meta = [c.size ? humanSize(c.size) : "", c.time ? new Date(c.time).toLocaleDateString() : ""]
|
||||
.filter(Boolean).join(" · ");
|
||||
}
|
||||
const he = heatFor(c.path, !!c.dir);
|
||||
if (he) {
|
||||
const dot = el(row, "span", "heatdot lvl" + heatLevel(he));
|
||||
dot.title = heatText(he) + " in 30 days";
|
||||
meta = heatText(he) + (meta ? " · " + meta : "");
|
||||
}
|
||||
el(row, "span", "dl-meta", meta);
|
||||
row.onclick = () => openPath(c.path);
|
||||
row.onkeydown = (e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); row.click(); } };
|
||||
@@ -964,6 +1030,8 @@ function showMeta(doc) {
|
||||
const parts = [];
|
||||
if (doc.author) parts.push(doc.author + (doc.device ? " on " + doc.device : ""));
|
||||
if (doc.time) parts.push(new Date(doc.time).toLocaleString());
|
||||
const he = heatMap && heatMap[doc.path];
|
||||
if (he && heatTotal(he)) parts.push(heatText(he) + " / 30d");
|
||||
$("meta").textContent = parts.join(" · ");
|
||||
}
|
||||
|
||||
|
||||
@@ -269,6 +269,12 @@ button, input, a.btn { font-family: inherit; }
|
||||
.dl-row:hover .ticon { color: var(--text-faint); }
|
||||
.dl-name { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 13.5px; color: var(--text); }
|
||||
.dl-meta { flex: none; font-size: 12px; color: var(--text-faint); font-variant-numeric: tabular-nums; }
|
||||
/* read-heat dot: intensity grows with 30-day reads (heatLevel in app.js) */
|
||||
.heatdot { flex: none; width: 7px; height: 7px; border-radius: 50%; background: var(--accent); }
|
||||
.heatdot.lvl1 { opacity: .3; }
|
||||
.heatdot.lvl2 { opacity: .55; }
|
||||
.heatdot.lvl3 { opacity: .8; }
|
||||
.heatdot.lvl4 { opacity: 1; box-shadow: 0 0 6px rgba(245, 166, 35, .55); }
|
||||
.dl-empty { padding: 24px 14px; color: var(--text-faint); font-size: 13px; border: 1px dashed var(--border); border-radius: var(--r-card); text-align: center; }
|
||||
.dl-h3 { margin: 28px 0 8px; font-size: 10.5px; text-transform: uppercase; letter-spacing: .07em; color: var(--text-ghost); font-weight: 600; }
|
||||
.dl-hlist { border: 1px solid var(--border); border-radius: var(--r-card); background: var(--bg-side); overflow: hidden; max-width: none; }
|
||||
|
||||
Reference in New Issue
Block a user