mirror of
https://github.com/runbear-io/beardrive.git
synced 2026-08-25 08:08:08 +02:00
Snow Lee (Sungwon)andGitHub
3de8590b6b
Nothing on the /store/* wire was compressed, while the corpus it carries is markdown and source. Compression lands as a pure transport concern: content addressing, the storage layout and the journal format all stay over the uncompressed bytes. The two legs are not symmetric. Pull needs no negotiation — net/http already sends Accept-Encoding: gzip and inflates transparently — so devices built before this get it the day the hub ships; a real pre-compression binary receives 19,958 bytes for a 148 KB corpus (7.4x) with no client change. Push is negotiated through sign()'s accept_encoding, because a gzip body posted to an old hub would be stored under the sha256 of its plaintext. The hub inflates ABOVE spool — the sha a key promises, the ops a journal carries and the size that gets billed are all plaintext properties — and the inflate is bounded at 256 MiB, because Content-Encoding severs the one-wire-byte-one-disk-byte relationship that made spool safe unbounded. The presigned direct-to-storage leg stays raw and is asserted to. Known deployment caveat: a compressed push clears ContentLength, so it goes out chunked where every push was sized before. A reverse proxy that buffers or rejects chunked request bodies would fail pushes (degrading to Offline and retrying, not losing data).
78 lines
2.1 KiB
Go
78 lines
2.1 KiB
Go
package remote
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// The probe consumes the bytes it judges, so the stream it hands back must be
|
|
// byte-identical to the one it was given — for a stream longer than the probe
|
|
// window, one shorter, and an empty one. A probe that eats bytes corrupts every
|
|
// push and pull that runs through it, and the damage surfaces as a sha
|
|
// mismatch nowhere near this file.
|
|
func TestCompressibleRejoinsTheStream(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
in []byte
|
|
want bool
|
|
}{
|
|
{"text past the window", []byte(strings.Repeat("package main // hello hello\n", 5000)), true},
|
|
{"text under the window", []byte(strings.Repeat("hello beardrive\n", 100)), true},
|
|
{"already compressed", randomBytes(200 << 10), false},
|
|
{"tiny", []byte("hi"), false},
|
|
{"empty", nil, false},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got, worth, err := Compressible(bytes.NewReader(c.in))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if worth != c.want {
|
|
t.Errorf("worth = %v, want %v", worth, c.want)
|
|
}
|
|
rejoined, err := io.ReadAll(got)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !bytes.Equal(rejoined, c.in) {
|
|
t.Fatalf("rejoined stream is %d bytes, want the original %d", len(rejoined), len(c.in))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAcceptsGzip(t *testing.T) {
|
|
cases := map[string]bool{
|
|
"": false,
|
|
"identity": false,
|
|
"gzip": true,
|
|
"deflate, gzip;q=1.0, *;q=0": true,
|
|
"GZIP": true,
|
|
"x-gzip": false, // a different token, not a prefix match
|
|
}
|
|
for hdr, want := range cases {
|
|
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
|
if hdr != "" {
|
|
r.Header.Set("Accept-Encoding", hdr)
|
|
}
|
|
if got := AcceptsGzip(r); got != want {
|
|
t.Errorf("AcceptsGzip(%q) = %v, want %v", hdr, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// randomBytes stands in for already-compressed content (JPEG, zip, model
|
|
// weights): incompressible by construction, which is the whole point.
|
|
func randomBytes(n int) []byte {
|
|
b := make([]byte, n)
|
|
rng := rand.New(rand.NewSource(1))
|
|
rng.Read(b)
|
|
return b
|
|
}
|