feat(zddc): GET /dir/?zip=1 — stream an ACL-filtered .zip of a subtree

zddc-server can now hand back a whole directory subtree as a single
streamed application/zip download: GET /some/dir/?zip=1 (works on both
/dir and /dir/) → Content-Type: application/zip + Content-Disposition:
attachment; filename="<dir>.zip", containing every readable file under
/some/dir/, recursively.

handler.ServeSubtreeZip walks the tree with filepath.WalkDir, ACL-gates
each file by the .zddc chain of its containing directory (per-dir
decision cache, same shape as serveArchiveListing), skips hidden
entries ("." and "_" prefixes — .zddc, _template, _app), and adds a
.zip *file* it encounters as opaque bytes (it does not recurse into it
— that's the navigable-virtual-surface feature, a different thing).
The response is streamed (zip.NewWriter straight onto the
ResponseWriter, Store for already-compressed extensions, Deflate
otherwise), so a fully-ACL-denied or empty subtree just yields a valid
empty zip rather than a 403 (a stream can't change status after the
headers go out; empty leaks no more than 403). HEAD sends the headers
and no body. The dispatch's directory ACL gate still runs first, so a
viewer who can't read the directory gets 403 before the handler.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
ZDDC 2026-05-12 12:59:17 -05:00
parent db1f44cf74
commit 81e065e5b0
4 changed files with 461 additions and 0 deletions

View file

@ -1048,6 +1048,12 @@ func dispatch(cfg config.Config, idx *archive.Index, ring *handler.LogRing, apps
if (r.Method == http.MethodGet || r.Method == http.MethodHead) &&
(strings.HasSuffix(urlPath, "/") || filepath.Ext(urlPath) == "") &&
zddc.IsDeclaredPath(cfg.Root, absPath) {
if r.URL.Query().Has("zip") {
// Subtree download of a cascade-declared dir that
// doesn't exist on disk yet → an empty zip.
handler.ServeSubtreeZip(cfg, w, r, absPath)
return
}
if strings.HasSuffix(urlPath, "/") {
handler.ServeDirectory(cfg, appsSrv, w, r)
return
@ -1079,6 +1085,15 @@ func dispatch(cfg config.Config, idx *archive.Index, ring *handler.LogRing, apps
return
}
}
// Subtree download: GET /dir/?zip=1 streams an application/zip of
// every readable file under this directory, ACL-filtered. Checked
// before the slash/no-slash routing so it works on both /dir and
// /dir/. Writes (PUT/DELETE/POST) never reach here — they're
// intercepted by the file API earlier — so this is GET/HEAD only.
if r.URL.Query().Has("zip") {
handler.ServeSubtreeZip(cfg, w, r, absPath)
return
}
// Slash/no-slash routing convention: trailing slash → the
// directory view (handler.ServeDirectory → DirTool, which
// resolves to browse by default; JSON requests always get the

View file

@ -2,6 +2,7 @@ package main
import (
"archive/zip"
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
@ -842,6 +843,79 @@ func mustWrite(t *testing.T, path, body string) {
}
}
// TestDispatchSubtreeZip exercises the `?zip=1` subtree-download hook:
// it routes to handler.ServeSubtreeZip on both the slash and no-slash
// forms of a directory URL, and the dispatch's directory ACL gate
// still applies (a viewer with no read access to the directory gets
// 403 before the zip handler runs).
func TestDispatchSubtreeZip(t *testing.T) {
root := t.TempDir()
mustWrite(t, filepath.Join(root, ".zddc"),
"acl:\n permissions:\n \"*\": r\n")
mustMkdir(t, filepath.Join(root, "Proj", "staging", "2025-01-15_AAA-EM-TRN-0001 (IFC) - T"))
mustWrite(t, filepath.Join(root, "Proj", "staging", "2025-01-15_AAA-EM-TRN-0001 (IFC) - T", "doc.txt"), "hello")
// A subtree only alice@x may read.
mustMkdir(t, filepath.Join(root, "Proj", "locked"))
mustWrite(t, filepath.Join(root, "Proj", "locked", ".zddc"),
"acl:\n inherit: false\n permissions:\n \"alice@x\": rwcda\n")
mustWrite(t, filepath.Join(root, "Proj", "locked", "secret.txt"), "s")
idx, err := archive.BuildIndex(root)
if err != nil {
t.Fatalf("BuildIndex: %v", err)
}
cfg := config.Config{Root: root, IndexPath: ".archive", EmailHeader: "X-Auth-Request-Email"}
ring := handler.NewLogRing(10)
appsSrv, err := setupApps(cfg)
if err != nil {
t.Fatalf("setupApps: %v", err)
}
do := func(path, email string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodGet, path, nil)
req = req.WithContext(context.WithValue(req.Context(), handler.EmailKey, email))
rec := httptest.NewRecorder()
dispatch(cfg, idx, ring, appsSrv, nil, rec, req)
return rec
}
for _, path := range []string{"/Proj/staging/?zip=1", "/Proj/staging?zip=1"} {
rec := do(path, "bob@x")
if rec.Code != http.StatusOK {
t.Fatalf("%s status=%d, want 200", path, rec.Code)
}
if ct := rec.Header().Get("Content-Type"); ct != "application/zip" {
t.Errorf("%s Content-Type=%q", path, ct)
}
if rec.Header().Get("X-ZDDC-Source") != "subtree-zip" {
t.Errorf("%s missing X-ZDDC-Source", path)
}
body := rec.Body.Bytes()
zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("%s body not a zip: %v", path, err)
}
var foundDoc bool
for _, f := range zr.File {
if strings.HasSuffix(f.Name, "/doc.txt") || f.Name == "staging/2025-01-15_AAA-EM-TRN-0001 (IFC) - T/doc.txt" {
foundDoc = true
}
}
if !foundDoc {
t.Errorf("%s zip missing doc.txt; entries=%d", path, len(zr.File))
}
}
// The dispatch's directory ACL gate runs before ServeSubtreeZip:
// bob@x can't read /Proj/locked at all → 403, no zip.
if rec := do("/Proj/locked/?zip=1", "bob@x"); rec.Code != http.StatusForbidden {
t.Errorf("bob@x /Proj/locked/?zip=1 status=%d, want 403", rec.Code)
}
// alice@x can → 200 zip.
if rec := do("/Proj/locked/?zip=1", "alice@x"); rec.Code != http.StatusOK {
t.Errorf("alice@x /Proj/locked/?zip=1 status=%d, want 200", rec.Code)
}
}
// TestGzhttpWrapper_CompressesLargeResponses asserts the gzhttp wrapper
// behavior we wire in main(): responses above MinSize get gzip-encoded
// when the client advertises Accept-Encoding: gzip; small responses

View file

@ -0,0 +1,162 @@
package handler
import (
"archive/zip"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"codeberg.org/VARASYS/ZDDC/zddc/internal/config"
"codeberg.org/VARASYS/ZDDC/zddc/internal/policy"
"codeberg.org/VARASYS/ZDDC/zddc/internal/zddc"
)
// alreadyCompressedExt is the set of file extensions whose contents are
// already compressed (or incompressible) — re-DEFLATE-ing them in the
// output zip just burns CPU in the response path for ~no size win, so
// they're stored verbatim instead.
var alreadyCompressedExt = map[string]bool{
".zip": true, ".gz": true, ".bz2": true, ".xz": true, ".7z": true,
".pdf": true,
".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true,
".tif": true, ".tiff": true,
".docx": true, ".xlsx": true, ".pptx": true, ".odt": true, ".ods": true,
".mp3": true, ".mp4": true, ".m4a": true, ".webm": true, ".avi": true, ".mov": true,
}
func zipMethodFor(name string) uint16 {
if alreadyCompressedExt[strings.ToLower(filepath.Ext(name))] {
return zip.Store
}
return zip.Deflate
}
// ServeSubtreeZip streams an application/zip download of every readable
// file under absDir (recursively), ACL-filtered against the requester.
// It's the handler behind `GET /some/dir/?zip=1`.
//
// Permissions: each file is gated by the .zddc chain of its containing
// directory (cached per directory), exactly like serveArchiveListing.
// Hidden entries — anything whose name starts with "." (.zddc, .archive
// is virtual anyway) or "_" (_template, _app) — are skipped, matching
// what the browse listing already hides. A `.zip` *file* found in the
// tree is added as opaque bytes (not recursed into; `…/Foo.zip/…` is a
// navigable surface elsewhere, but a subtree download just bundles the
// archive as-is).
//
// The response is streamed: headers go out first, then the zip is
// written entry-by-entry. So we can't 403-after-the-fact when the
// caller can read nothing under absDir — they just get a valid empty
// zip. (Empty leaks no more than a 403 would.) absDir need not exist
// on disk (a cascade-declared-but-unmaterialised folder → empty zip).
func ServeSubtreeZip(cfg config.Config, w http.ResponseWriter, r *http.Request, absDir string) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
w.Header().Set("Allow", "GET, HEAD")
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
return
}
zipName := filepath.Base(absDir) + ".zip"
prefix := filepath.Base(absDir) // top-level folder name inside the zip
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", "attachment; filename=\""+sanitizeFilename(zipName)+"\"")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-ZDDC-Source", "subtree-zip")
if r.Method == http.MethodHead {
return
}
email := EmailFromContext(r)
decider := DeciderFromContext(r)
ctx := r.Context()
// Per-directory ACL-decision cache (same shape as serveArchiveListing).
aclCache := make(map[string]bool)
allowed := func(fileDir string) bool {
if v, ok := aclCache[fileDir]; ok {
return v
}
chain, err := zddc.EffectivePolicy(cfg.Root, fileDir)
if err != nil {
aclCache[fileDir] = false
return false
}
rel, relErr := filepath.Rel(cfg.Root, fileDir)
urlPath := "/"
if relErr == nil && rel != "." {
urlPath = "/" + filepath.ToSlash(rel)
}
v, _ := policy.AllowFromChain(ctx, decider, chain, email, urlPath)
aclCache[fileDir] = v
return v
}
zw := zip.NewWriter(w)
walkErr := filepath.WalkDir(absDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil // skip unreadable entries; covers absDir-doesn't-exist
}
name := d.Name()
if d.IsDir() {
if path != absDir && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_")) {
return filepath.SkipDir
}
return nil
}
if !d.Type().IsRegular() {
return nil // skip symlinks, devices, etc.
}
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") {
return nil
}
if !allowed(filepath.Dir(path)) {
return nil
}
rel, relErr := filepath.Rel(absDir, path)
if relErr != nil {
return nil
}
info, infoErr := d.Info()
hdr := &zip.FileHeader{
Name: prefix + "/" + filepath.ToSlash(rel),
Method: zipMethodFor(name),
}
if infoErr == nil {
hdr.Modified = info.ModTime()
}
entry, cErr := zw.CreateHeader(hdr)
if cErr != nil {
return cErr // writer/connection is broken — stop the walk
}
f, oErr := os.Open(path)
if oErr != nil {
slog.Warn("subtree-zip: open file", "path", path, "err", oErr)
return nil // best-effort; stream already in flight
}
_, copyErr := io.Copy(entry, f)
f.Close()
if copyErr != nil {
slog.Warn("subtree-zip: copy file", "path", path, "err", copyErr)
return copyErr // connection likely gone — stop
}
return nil
})
if walkErr != nil {
slog.Warn("subtree-zip: walk aborted", "dir", absDir, "err", walkErr)
}
if err := zw.Close(); err != nil {
slog.Warn("subtree-zip: close writer", "dir", absDir, "err", err)
}
}
// sanitizeFilename strips characters that would break a quoted
// Content-Disposition filename (CR/LF/quote/backslash) — directory
// basenames almost never contain these, but be defensive.
func sanitizeFilename(s string) string {
return strings.NewReplacer("\r", "", "\n", "", `"`, "", `\`, "").Replace(s)
}

View file

@ -0,0 +1,210 @@
package handler
import (
"archive/zip"
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sort"
"testing"
"codeberg.org/VARASYS/ZDDC/zddc/internal/config"
)
func mkfile(t *testing.T, path, body string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
// readZipResponse decodes the zip body of a recorded response into
// name → bytes.
func readZipResponse(t *testing.T, rec *httptest.ResponseRecorder) map[string][]byte {
t.Helper()
body := rec.Body.Bytes()
if len(body) == 0 {
return map[string][]byte{}
}
zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
t.Fatalf("response body is not a valid zip: %v (%d bytes)", err, len(body))
}
out := map[string][]byte{}
for _, f := range zr.File {
rc, err := f.Open()
if err != nil {
t.Fatalf("open zip entry %q: %v", f.Name, err)
}
b, _ := io.ReadAll(rc)
rc.Close()
out[f.Name] = b
}
return out
}
func subtreeReq(t *testing.T, method, urlPath, email string) *http.Request {
t.Helper()
req := httptest.NewRequest(method, urlPath, nil)
req = req.WithContext(context.WithValue(req.Context(), EmailKey, email))
return req
}
func TestServeSubtreeZip(t *testing.T) {
root := t.TempDir()
mkfile(t, filepath.Join(root, ".zddc"), "acl:\n permissions:\n \"*\": r\n")
sub := filepath.Join(root, "Proj", "sub")
mkfile(t, filepath.Join(sub, "a.txt"), "AAA")
mkfile(t, filepath.Join(sub, "nested", "b.txt"), "BBB")
mkfile(t, filepath.Join(sub, ".zddc"), "acl:\n permissions:\n \"*\": r\n") // hidden — must not appear
mkfile(t, filepath.Join(sub, "_template", "t.txt"), "tmpl") // hidden dir — must not appear
// A locked subdir only owner@x can read.
mkfile(t, filepath.Join(sub, "secret", ".zddc"), "acl:\n inherit: false\n permissions:\n \"owner@x\": rwcda\n")
mkfile(t, filepath.Join(sub, "secret", "s.txt"), "SECRET")
// A .zip file inside the subtree — must appear as opaque bytes, not extracted.
writeTestZip(t, filepath.Join(sub, "Pack.zip"), map[string]string{"inner.txt": "INNER"})
cfg := config.Config{Root: root}
t.Run("headers + ACL-filtered contents", func(t *testing.T) {
req := subtreeReq(t, http.MethodGet, "/Proj/sub/?zip=1", "bob@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, sub)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%d bytes", rec.Code, rec.Body.Len())
}
if ct := rec.Header().Get("Content-Type"); ct != "application/zip" {
t.Errorf("Content-Type=%q", ct)
}
if cd := rec.Header().Get("Content-Disposition"); cd != `attachment; filename="sub.zip"` {
t.Errorf("Content-Disposition=%q", cd)
}
if src := rec.Header().Get("X-ZDDC-Source"); src != "subtree-zip" {
t.Errorf("X-ZDDC-Source=%q", src)
}
if cc := rec.Header().Get("Cache-Control"); cc != "no-store" {
t.Errorf("Cache-Control=%q", cc)
}
got := readZipResponse(t, rec)
var names []string
for n := range got {
names = append(names, n)
}
sort.Strings(names)
want := []string{"sub/Pack.zip", "sub/a.txt", "sub/nested/b.txt"}
if len(names) != len(want) {
t.Fatalf("zip entries = %v, want %v", names, want)
}
for i := range want {
if names[i] != want[i] {
t.Fatalf("zip entries = %v, want %v", names, want)
}
}
if string(got["sub/a.txt"]) != "AAA" {
t.Errorf("sub/a.txt = %q", got["sub/a.txt"])
}
// Pack.zip is opaque: its bytes are themselves a valid zip with inner.txt.
inner, err := zip.NewReader(bytes.NewReader(got["sub/Pack.zip"]), int64(len(got["sub/Pack.zip"])))
if err != nil {
t.Fatalf("sub/Pack.zip is not a zip: %v", err)
}
if len(inner.File) != 1 || inner.File[0].Name != "inner.txt" {
t.Errorf("sub/Pack.zip should contain just inner.txt; got %d entries", len(inner.File))
}
})
t.Run("owner sees the locked subdir too", func(t *testing.T) {
req := subtreeReq(t, http.MethodGet, "/Proj/sub/?zip=1", "owner@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, sub)
got := readZipResponse(t, rec)
if _, ok := got["sub/secret/s.txt"]; !ok {
t.Errorf("owner@x should see sub/secret/s.txt; got %v", keysOf(got))
}
if string(got["sub/secret/s.txt"]) != "SECRET" {
t.Errorf("sub/secret/s.txt = %q", got["sub/secret/s.txt"])
}
})
t.Run("HEAD sets headers, no body", func(t *testing.T) {
req := subtreeReq(t, http.MethodHead, "/Proj/sub/?zip=1", "bob@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, sub)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d", rec.Code)
}
if rec.Header().Get("Content-Type") != "application/zip" {
t.Errorf("missing Content-Type on HEAD")
}
if rec.Body.Len() != 0 {
t.Errorf("HEAD body should be empty; got %d bytes", rec.Body.Len())
}
})
t.Run("method not allowed", func(t *testing.T) {
req := subtreeReq(t, http.MethodPost, "/Proj/sub/?zip=1", "bob@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, sub)
if rec.Code != http.StatusMethodNotAllowed {
t.Errorf("POST status=%d, want 405", rec.Code)
}
})
}
func TestServeSubtreeZip_AllDenied(t *testing.T) {
root := t.TempDir()
mkfile(t, filepath.Join(root, ".zddc"), "acl:\n permissions:\n \"*\": r\n")
locked := filepath.Join(root, "Proj", "locked")
mkfile(t, filepath.Join(locked, ".zddc"), "acl:\n inherit: false\n permissions:\n \"owner@x\": rwcda\n")
mkfile(t, filepath.Join(locked, "x.txt"), "x")
mkfile(t, filepath.Join(locked, "deep", "y.txt"), "y")
cfg := config.Config{Root: root}
req := subtreeReq(t, http.MethodGet, "/Proj/locked/?zip=1", "bob@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, locked)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d, want 200 (empty zip)", rec.Code)
}
got := readZipResponse(t, rec)
if len(got) != 0 {
t.Errorf("expected empty zip for a fully-denied subtree; got %v", keysOf(got))
}
}
func TestServeSubtreeZip_Nonexistent(t *testing.T) {
root := t.TempDir()
mkfile(t, filepath.Join(root, ".zddc"), "acl:\n permissions:\n \"*\": r\n")
cfg := config.Config{Root: root}
missing := filepath.Join(root, "Proj", "working") // declared by the cascade, not on disk
req := subtreeReq(t, http.MethodGet, "/Proj/working/?zip=1", "bob@x")
rec := httptest.NewRecorder()
ServeSubtreeZip(cfg, rec, req, missing)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d, want 200 (empty zip)", rec.Code)
}
if got := readZipResponse(t, rec); len(got) != 0 {
t.Errorf("expected empty zip for a nonexistent dir; got %v", keysOf(got))
}
if cd := rec.Header().Get("Content-Disposition"); cd != `attachment; filename="working.zip"` {
t.Errorf("Content-Disposition=%q", cd)
}
}
func keysOf(m map[string][]byte) []string {
var ks []string
for k := range m {
ks = append(ks, k)
}
sort.Strings(ks)
return ks
}