mirror of
https://github.com/runbear-io/beardrive.git
synced 2026-08-25 08:08:08 +02:00
sfs v0.1: mountable synced file system for AI agents
- sfs mnt/umnt/sync/status/log/remote/whoami CLI (cobra) - per-device append-only journals + content-addressed blob store - deterministic lamport-ordered merge, LWW with conflict-copy preservation - cloud-agnostic backends: S3, GCS, file:// (S3-compatible via AWS_ENDPOINT_URL) - offline-first: real files on disk, journal locally, push on reconnect - background sync daemon per mount with change tracking (device/author/time) - macOS + Linux; Homebrew formula + goreleaser release pipeline Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
snow
@@ -0,0 +1,255 @@
|
||||
// Package store manages a volume's local on-disk state: a content-addressed
|
||||
// blob store, the per-device journals, and small JSON state files. Everything
|
||||
// a volume needs works offline; the remote is only used to exchange blobs and
|
||||
// journals.
|
||||
//
|
||||
// Layout under <sfs home>/volumes/<volume>/:
|
||||
//
|
||||
// blobs/<aa>/<sha256> content-addressed file contents (immutable)
|
||||
// journal/<device>.jsonl per-device op logs (own + cached copies of peers)
|
||||
// state.json what is currently materialized in the folder
|
||||
// sync.json lamport clock + push cursor
|
||||
// lock flock guarding cycles
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/runbear-io/sfs/internal/journal"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func Open(dir string) (*Store, error) {
|
||||
for _, d := range []string{dir, filepath.Join(dir, "blobs"), filepath.Join(dir, "journal"), filepath.Join(dir, "tmp")} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Store{dir: dir}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Dir() string { return s.dir }
|
||||
func (s *Store) tmpDir() string { return filepath.Join(s.dir, "tmp") }
|
||||
|
||||
// ---- blobs ----
|
||||
|
||||
func (s *Store) BlobPath(sum string) string {
|
||||
return filepath.Join(s.dir, "blobs", sum[:2], sum)
|
||||
}
|
||||
|
||||
func (s *Store) HasBlob(sum string) bool {
|
||||
if len(sum) < 3 {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(s.BlobPath(sum))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// PutBlobReader streams r into the blob store, returning its sha256 and size.
|
||||
func (s *Store) PutBlobReader(r io.Reader) (string, int64, error) {
|
||||
tmp, err := os.CreateTemp(s.tmpDir(), "blob-*")
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(tmp, h), r)
|
||||
if cerr := tmp.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
sum := hex.EncodeToString(h.Sum(nil))
|
||||
if s.HasBlob(sum) {
|
||||
return sum, n, nil
|
||||
}
|
||||
dst := s.BlobPath(sum)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), dst); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return sum, n, nil
|
||||
}
|
||||
|
||||
func (s *Store) PutBlobFile(path string) (string, int64, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
return s.PutBlobReader(f)
|
||||
}
|
||||
|
||||
func (s *Store) PutBlobBytes(b []byte) (string, int64, error) {
|
||||
return s.PutBlobReader(strings.NewReader(string(b)))
|
||||
}
|
||||
|
||||
func (s *Store) OpenBlob(sum string) (*os.File, error) {
|
||||
return os.Open(s.BlobPath(sum))
|
||||
}
|
||||
|
||||
// ---- journals ----
|
||||
|
||||
func (s *Store) JournalPath(device string) string {
|
||||
return filepath.Join(s.dir, "journal", device+".jsonl")
|
||||
}
|
||||
|
||||
func (s *Store) AppendOps(device string, ops []journal.Op) error {
|
||||
return journal.Append(s.JournalPath(device), ops)
|
||||
}
|
||||
|
||||
func (s *Store) DeviceOps(device string) ([]journal.Op, error) {
|
||||
return journal.ReadFile(s.JournalPath(device))
|
||||
}
|
||||
|
||||
// Devices lists device IDs that have a local journal copy.
|
||||
func (s *Store) Devices() ([]string, error) {
|
||||
entries, err := os.ReadDir(filepath.Join(s.dir, "journal"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".jsonl") {
|
||||
out = append(out, strings.TrimSuffix(e.Name(), ".jsonl"))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AllOps returns the union of every journal known locally.
|
||||
func (s *Store) AllOps() ([]journal.Op, error) {
|
||||
devs, err := s.Devices()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var all []journal.Op
|
||||
for _, d := range devs {
|
||||
ops, err := s.DeviceOps(d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("journal %s: %w", d, err)
|
||||
}
|
||||
all = append(all, ops...)
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// ---- materialized-state cache (state.json) ----
|
||||
|
||||
// CachedFile records what sfs last wrote to / observed in the working folder
|
||||
// for a path. Size+MTimeNS make change detection cheap; Blob ties it back to
|
||||
// content.
|
||||
type CachedFile struct {
|
||||
Blob string `json:"blob"`
|
||||
Size int64 `json:"size"`
|
||||
Mode uint32 `json:"mode"`
|
||||
MTimeNS int64 `json:"mtime_ns"`
|
||||
}
|
||||
|
||||
func (s *Store) LoadCache() (map[string]CachedFile, error) {
|
||||
out := map[string]CachedFile{}
|
||||
if err := readJSON(filepath.Join(s.dir, "state.json"), &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveCache(c map[string]CachedFile) error {
|
||||
return WriteJSONAtomic(filepath.Join(s.dir, "state.json"), c)
|
||||
}
|
||||
|
||||
// ---- sync state (sync.json) ----
|
||||
|
||||
type SyncState struct {
|
||||
Lamport int64 `json:"lamport"`
|
||||
PushedOps int64 `json:"pushed_ops"` // how many of our own ops the remote has
|
||||
}
|
||||
|
||||
func (s *Store) LoadSync() (SyncState, error) {
|
||||
var st SyncState
|
||||
if err := readJSON(filepath.Join(s.dir, "sync.json"), &st); err != nil {
|
||||
return st, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveSync(st SyncState) error {
|
||||
return WriteJSONAtomic(filepath.Join(s.dir, "sync.json"), st)
|
||||
}
|
||||
|
||||
// ---- locking ----
|
||||
|
||||
// Lock takes an exclusive flock for the volume, serializing sync cycles
|
||||
// between the daemon and one-shot commands. Blocks until acquired.
|
||||
func (s *Store) Lock() (func() error, error) {
|
||||
f, err := os.OpenFile(filepath.Join(s.dir, "lock"), os.O_CREATE|os.O_RDWR, 0o644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return func() error {
|
||||
defer f.Close()
|
||||
return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ---- small JSON helpers ----
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
|
||||
// WriteJSONAtomic writes v as JSON via temp-file + rename.
|
||||
func WriteJSONAtomic(path string, v any) error {
|
||||
data, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return WriteFileAtomic(path, data, 0o644)
|
||||
}
|
||||
|
||||
// WriteFileAtomic writes data via a temp file in the same directory + rename.
|
||||
func WriteFileAtomic(path string, data []byte, mode os.FileMode) error {
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".sfs-tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), mode); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/runbear-io/sfs/internal/journal"
|
||||
)
|
||||
|
||||
func TestBlobRoundtrip(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum, n, err := s.PutBlobBytes([]byte("hello sfs"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 9 {
|
||||
t.Fatalf("size = %d, want 9", n)
|
||||
}
|
||||
if !s.HasBlob(sum) {
|
||||
t.Fatal("blob not stored")
|
||||
}
|
||||
// dedupe: same content, same sum, no error
|
||||
sum2, _, err := s.PutBlobBytes([]byte("hello sfs"))
|
||||
if err != nil || sum2 != sum {
|
||||
t.Fatalf("dedupe failed: %v %v", sum2, err)
|
||||
}
|
||||
f, err := s.OpenBlob(sum)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
data := make([]byte, 16)
|
||||
k, _ := f.Read(data)
|
||||
if string(data[:k]) != "hello sfs" {
|
||||
t.Fatalf("content mismatch: %q", data[:k])
|
||||
}
|
||||
}
|
||||
|
||||
func TestJournalAndState(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ops := []journal.Op{{Seq: 1, Lamport: 1, Device: "devA", Kind: journal.KindPut, Path: "f", Blob: "b"}}
|
||||
if err := s.AppendOps("devA", ops); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AppendOps("devB", []journal.Op{{Seq: 1, Lamport: 2, Device: "devB", Kind: journal.KindDelete, Path: "f"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := s.AllOps()
|
||||
if err != nil || len(all) != 2 {
|
||||
t.Fatalf("AllOps = %v, %v", all, err)
|
||||
}
|
||||
devs, _ := s.Devices()
|
||||
if len(devs) != 2 {
|
||||
t.Fatalf("Devices = %v", devs)
|
||||
}
|
||||
|
||||
cache := map[string]CachedFile{"f": {Blob: "b", Size: 1, MTimeNS: 42}}
|
||||
if err := s.SaveCache(cache); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.LoadCache()
|
||||
if err != nil || got["f"].MTimeNS != 42 {
|
||||
t.Fatalf("cache roundtrip: %v %v", got, err)
|
||||
}
|
||||
|
||||
st := SyncState{Lamport: 7, PushedOps: 3}
|
||||
if err := s.SaveSync(st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotSt, err := s.LoadSync()
|
||||
if err != nil || gotSt != st {
|
||||
t.Fatalf("sync state roundtrip: %v %v", gotSt, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLock(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unlock, err := s.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// re-acquirable after unlock
|
||||
unlock2, err := s.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unlock2()
|
||||
}
|
||||
|
||||
func TestWriteFileAtomic(t *testing.T) {
|
||||
p := t.TempDir() + "/x.json"
|
||||
if err := WriteFileAtomic(p, []byte("data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil || string(b) != "data" {
|
||||
t.Fatalf("got %q %v", b, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user