claude-timemachine
2b7290626e
The deferred 'hardlink blobs from tarball' optimization from DESIGN.md
landed as 'just walk the tarball and write blobs separately' for v1.
GET /v1/blob/{sha} was 404'ing because the blob store was empty —
storage only had snapshots/<id>.tar.zst and a manifest.
Server now:
1. Parses uploaded multipart manifest + tarball
2. Walks the tar entries, computes each entry's sha256
3. Cross-checks against the manifest's declared sha (rejects 400 on mismatch)
4. Writes each blob to <user>/blobs/ via Storage.WriteBlob
5. Then stores the snapshot tarball + manifest as before
2 new tests cover: (a) POST then GET /v1/blob/{sha} round-trip,
(b) manifest-claims-different-sha-than-tarball rejection.
Discovered via e2e smoke against frazclient: pull 404'd on every blob
after a successful push. 33/33 tests pass.
412 lines
12 KiB
Go
412 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// buildTarball wraps the given files into an in-memory tar archive. Used by
|
|
// the POST snapshot tests; mirrors what the real client emits.
|
|
func buildTarball(t *testing.T, files map[string][]byte) []byte {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
tw := tar.NewWriter(&buf)
|
|
for name, data := range files {
|
|
hdr := &tar.Header{Name: name, Mode: 0o600, Size: int64(len(data))}
|
|
if err := tw.WriteHeader(hdr); err != nil {
|
|
t.Fatalf("tar header %s: %v", name, err)
|
|
}
|
|
if _, err := tw.Write(data); err != nil {
|
|
t.Fatalf("tar write %s: %v", name, err)
|
|
}
|
|
}
|
|
if err := tw.Close(); err != nil {
|
|
t.Fatalf("tar close: %v", err)
|
|
}
|
|
return buf.Bytes()
|
|
}
|
|
|
|
// scopedVerifier accepts a fixed token and reports a fixed user + scope set.
|
|
type scopedVerifier struct {
|
|
token string
|
|
user string
|
|
scopes []string
|
|
err error
|
|
}
|
|
|
|
func (v *scopedVerifier) Verify(_ context.Context, token string) (*AuthInfo, error) {
|
|
if v.err != nil {
|
|
return nil, v.err
|
|
}
|
|
if token != v.token {
|
|
return nil, ErrUnauthenticated
|
|
}
|
|
return &AuthInfo{User: v.user, Scopes: v.scopes}, nil
|
|
}
|
|
|
|
func newTestServer(t *testing.T) (*Server, *Storage, *scopedVerifier) {
|
|
t.Helper()
|
|
st := newStorage(t)
|
|
v := &scopedVerifier{
|
|
token: "good-token",
|
|
user: "user1",
|
|
scopes: []string{"cloud:rw"},
|
|
}
|
|
return NewServer(st, v, DefaultQuota(1<<20 /* 1 MB */)), st, v
|
|
}
|
|
|
|
func authReq(t *testing.T, method, path string, body io.Reader) *http.Request {
|
|
t.Helper()
|
|
r := httptest.NewRequest(method, path, body)
|
|
r.Header.Set("Authorization", "Bearer good-token")
|
|
return r
|
|
}
|
|
|
|
func TestServer_Auth_MissingBearer_401(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusUnauthorized {
|
|
t.Errorf("got %d, want 401", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_Auth_BadToken_401(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
r := httptest.NewRequest(http.MethodGet, "/v1/manifest", nil)
|
|
r.Header.Set("Authorization", "Bearer wrong-token")
|
|
s.ServeHTTP(rec, r)
|
|
if rec.Code != http.StatusUnauthorized {
|
|
t.Errorf("got %d, want 401", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_Auth_Revoked_403(t *testing.T) {
|
|
s, _, v := newTestServer(t)
|
|
v.err = ErrRevoked
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusForbidden {
|
|
t.Errorf("got %d, want 403", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_Auth_MissingScope_403(t *testing.T) {
|
|
s, _, v := newTestServer(t)
|
|
v.scopes = []string{"some-other-scope"}
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusForbidden {
|
|
t.Errorf("got %d, want 403", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_Auth_BackendError_502(t *testing.T) {
|
|
s, _, v := newTestServer(t)
|
|
v.err = errors.New("upstream down")
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusBadGateway {
|
|
t.Errorf("got %d, want 502", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_GetManifest_NoSnapshots_204(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusNoContent {
|
|
t.Errorf("got %d, want 204", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_PostSnapshot_RoundTrip(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
|
|
blob := []byte("file contents")
|
|
tarBytes := buildTarball(t, map[string][]byte{"options.txt": blob})
|
|
|
|
manifest := &Manifest{
|
|
SnapshotID: "01TESTSNAPSHOT0001",
|
|
CreatedAt: time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC),
|
|
Files: map[string]FileEntry{
|
|
"options.txt": {
|
|
SHA256: HashBytes(blob),
|
|
Size: int64(len(blob)),
|
|
Mtime: time.Date(2026, 6, 2, 11, 0, 0, 0, time.UTC),
|
|
},
|
|
},
|
|
}
|
|
manifestJSON, _ := json.Marshal(manifest)
|
|
|
|
body := &bytes.Buffer{}
|
|
mw := multipart.NewWriter(body)
|
|
fw, _ := mw.CreateFormFile("manifest", "manifest.json")
|
|
_, _ = fw.Write(manifestJSON)
|
|
fw, _ = mw.CreateFormFile("tarball", "snapshot.tar")
|
|
_, _ = fw.Write(tarBytes)
|
|
mw.Close()
|
|
|
|
req := authReq(t, http.MethodPost, "/v1/snapshot", body)
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("got %d, body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
var resp map[string]string
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
if resp["snapshot_id"] != "01TESTSNAPSHOT0001" {
|
|
t.Errorf("snapshot_id: got %s", resp["snapshot_id"])
|
|
}
|
|
|
|
// Now GET /v1/manifest should return the same
|
|
rec = httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/manifest", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("manifest after push: %d", rec.Code)
|
|
}
|
|
var got Manifest
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
|
if got.SnapshotID != "01TESTSNAPSHOT0001" {
|
|
t.Errorf("manifest snapshot id: got %s", got.SnapshotID)
|
|
}
|
|
}
|
|
|
|
func TestServer_GetBlob_ReturnsContent(t *testing.T) {
|
|
s, st, _ := newTestServer(t)
|
|
sha, _ := st.WriteBlob("user1", []byte("blob-payload"))
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/blob/"+sha, nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("got %d", rec.Code)
|
|
}
|
|
if rec.Body.String() != "blob-payload" {
|
|
t.Errorf("body: got %q", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
// Regression: after POST /v1/snapshot, GET /v1/blob/{sha} must succeed for
|
|
// every file declared in the uploaded manifest. We discovered during e2e
|
|
// smoke that the server was storing the tarball + manifest but not
|
|
// populating the blob store, so subsequent pulls 404'd.
|
|
func TestServer_PostSnapshot_PopulatesBlobStore(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
|
|
content := []byte("hello cloud")
|
|
sha := HashBytes(content)
|
|
tarBytes := buildTarball(t, map[string][]byte{"options.txt": content})
|
|
|
|
manifest := &Manifest{
|
|
SnapshotID: "01EXTRACTTESTABCDEF",
|
|
CreatedAt: time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC),
|
|
Files: map[string]FileEntry{
|
|
"options.txt": {
|
|
SHA256: sha,
|
|
Size: int64(len(content)),
|
|
Mtime: time.Date(2026, 6, 2, 11, 0, 0, 0, time.UTC),
|
|
},
|
|
},
|
|
}
|
|
manifestJSON, _ := json.Marshal(manifest)
|
|
|
|
body := &bytes.Buffer{}
|
|
mw := multipart.NewWriter(body)
|
|
fw, _ := mw.CreateFormFile("manifest", "manifest.json")
|
|
_, _ = fw.Write(manifestJSON)
|
|
fw, _ = mw.CreateFormFile("tarball", "snapshot.tar")
|
|
_, _ = fw.Write(tarBytes)
|
|
mw.Close()
|
|
|
|
req := authReq(t, http.MethodPost, "/v1/snapshot", body)
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("upload: got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
// Now GET /v1/blob/{sha} should succeed and return the original content
|
|
rec = httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/blob/"+sha, nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("blob fetch: got %d", rec.Code)
|
|
}
|
|
if !bytes.Equal(rec.Body.Bytes(), content) {
|
|
t.Errorf("blob content mismatch: got %q want %q", rec.Body.String(), content)
|
|
}
|
|
}
|
|
|
|
// Reject uploads whose tarball contents don't match the manifest's claimed sha.
|
|
func TestServer_PostSnapshot_RejectsManifestMismatch(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
tarBytes := buildTarball(t, map[string][]byte{"options.txt": []byte("ACTUAL")})
|
|
|
|
manifest := &Manifest{
|
|
SnapshotID: "01MISMATCHTEST00001",
|
|
CreatedAt: time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC),
|
|
Files: map[string]FileEntry{
|
|
"options.txt": {
|
|
SHA256: HashBytes([]byte("CLAIMED")), // lies about content
|
|
Size: 6,
|
|
Mtime: time.Date(2026, 6, 2, 11, 0, 0, 0, time.UTC),
|
|
},
|
|
},
|
|
}
|
|
manifestJSON, _ := json.Marshal(manifest)
|
|
|
|
body := &bytes.Buffer{}
|
|
mw := multipart.NewWriter(body)
|
|
fw, _ := mw.CreateFormFile("manifest", "manifest.json")
|
|
_, _ = fw.Write(manifestJSON)
|
|
fw, _ = mw.CreateFormFile("tarball", "snapshot.tar")
|
|
_, _ = fw.Write(tarBytes)
|
|
mw.Close()
|
|
|
|
req := authReq(t, http.MethodPost, "/v1/snapshot", body)
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Errorf("got %d, want 400; body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestServer_GetBlob_404(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
missing := strings.Repeat("0", 64)
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/blob/"+missing, nil))
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Errorf("got %d, want 404", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_PostSnapshot_BadManifest_400(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
body := &bytes.Buffer{}
|
|
mw := multipart.NewWriter(body)
|
|
fw, _ := mw.CreateFormFile("manifest", "manifest.json")
|
|
_, _ = fw.Write([]byte(`{"snapshot_id":"","created_at":"2026-01-01T00:00:00Z","files":{}}`))
|
|
fw, _ = mw.CreateFormFile("tarball", "snapshot.tar")
|
|
_, _ = fw.Write(buildTarball(t, nil))
|
|
mw.Close()
|
|
req := authReq(t, http.MethodPost, "/v1/snapshot", body)
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Errorf("got %d, want 400", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_PostSnapshot_QuotaExceeded_413(t *testing.T) {
|
|
st := newStorage(t)
|
|
v := &scopedVerifier{token: "good-token", user: "u", scopes: []string{"cloud:rw"}}
|
|
s := NewServer(st, v, DefaultQuota(100)) // 100 bytes
|
|
|
|
manifest := &Manifest{
|
|
SnapshotID: "01QUOTATEST00000001",
|
|
CreatedAt: time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC),
|
|
Files: map[string]FileEntry{
|
|
"f.txt": {
|
|
SHA256: HashBytes([]byte("x")),
|
|
Size: 1,
|
|
Mtime: time.Date(2026, 6, 2, 11, 0, 0, 0, time.UTC),
|
|
},
|
|
},
|
|
}
|
|
manifestJSON, _ := json.Marshal(manifest)
|
|
|
|
// Minimal tar with one 1-byte entry is already 1536+ bytes (header+data+EOF blocks),
|
|
// well over the 100-byte quota set in this test.
|
|
tarOver := buildTarball(t, map[string][]byte{"f.txt": []byte("x")})
|
|
body := &bytes.Buffer{}
|
|
mw := multipart.NewWriter(body)
|
|
fw, _ := mw.CreateFormFile("manifest", "manifest.json")
|
|
_, _ = fw.Write(manifestJSON)
|
|
fw, _ = mw.CreateFormFile("tarball", "tar")
|
|
_, _ = fw.Write(tarOver)
|
|
mw.Close()
|
|
|
|
req := authReq(t, http.MethodPost, "/v1/snapshot", body)
|
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusRequestEntityTooLarge {
|
|
t.Errorf("got %d, want 413", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestServer_ListSnapshots(t *testing.T) {
|
|
s, st, _ := newTestServer(t)
|
|
sha, _ := st.WriteBlob("user1", []byte("y"))
|
|
for _, id := range []string{"01AAA", "01BBB"} {
|
|
m := buildManifest(id, sha)
|
|
if err := st.StoreSnapshot("user1", m, []byte("t")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/snapshots", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("got %d", rec.Code)
|
|
}
|
|
var resp struct {
|
|
Snapshots []SnapshotInfo `json:"snapshots"`
|
|
}
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
if len(resp.Snapshots) != 2 {
|
|
t.Errorf("snapshots: got %d, want 2", len(resp.Snapshots))
|
|
}
|
|
}
|
|
|
|
func TestServer_GetQuota(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodGet, "/v1/quota", nil))
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("got %d", rec.Code)
|
|
}
|
|
var resp map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
if _, ok := resp["limit_bytes"]; !ok {
|
|
t.Errorf("missing limit_bytes; body=%s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestServer_Healthz(t *testing.T) {
|
|
s, _, _ := newTestServer(t)
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
|
if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
|
|
t.Errorf("got code=%d body=%q", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestServer_DeleteSnapshot_RefusesLatest_400(t *testing.T) {
|
|
s, st, _ := newTestServer(t)
|
|
sha, _ := st.WriteBlob("user1", []byte("z"))
|
|
m := buildManifest("01ONLYSNAPSHOT00001", sha)
|
|
if err := st.StoreSnapshot("user1", m, []byte("tar")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
s.ServeHTTP(rec, authReq(t, http.MethodDelete, "/v1/snapshot/01ONLYSNAPSHOT00001", nil))
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Errorf("got %d, want 400", rec.Code)
|
|
}
|
|
}
|