Git backed by object storage because you can't stop me
git object-storage kefka
10

Configure Feed

Select the types of activity you want to include in your feed.

feat(protocol): add git upload-pack support via git protocol

Implement native git protocol support in objgitd daemon. Enables clients
to use standard git commands (git clone, git fetch) over the git protocol
instead of requiring HTTP. Includes proper request handling and response
formatting for upload-pack operations.

Assisted-by: Claude Haiku 4.5 via Claude Code
Signed-off-by: Xe Iaso <me@xeiaso.net>

Xe Iaso (May 27, 2026, 9:52 PM EDT) ca89e095 415bbc79

+601 -62
+2
.gitignore
··· 1 + .env 2 + /objgitd
+161
cmd/objgitd/daemon.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "errors" 6 + "fmt" 7 + "io" 8 + "log/slog" 9 + "net" 10 + "net/url" 11 + "strings" 12 + "time" 13 + 14 + "github.com/go-git/go-billy/v6" 15 + git "github.com/go-git/go-git/v6" 16 + "github.com/go-git/go-git/v6/plumbing" 17 + "github.com/go-git/go-git/v6/plumbing/cache" 18 + "github.com/go-git/go-git/v6/plumbing/format/pktline" 19 + "github.com/go-git/go-git/v6/plumbing/protocol/packp" 20 + "github.com/go-git/go-git/v6/plumbing/transport" 21 + "github.com/go-git/go-git/v6/storage" 22 + "github.com/go-git/go-git/v6/storage/filesystem" 23 + ) 24 + 25 + // handshakeTimeout bounds how long a client has to send its git-proto-request. 26 + // It is cleared once the (possibly long) transfer begins. 27 + const handshakeTimeout = 30 * time.Second 28 + 29 + // daemon serves the git:// (TCP) protocol out of a billy filesystem. 30 + type daemon struct { 31 + fs billy.Filesystem 32 + loader transport.Loader 33 + allowPush bool 34 + } 35 + 36 + // plainStorer hides the underlying storer's PackfileWriter implementation. 37 + // 38 + // transport.ReceivePack stores an incoming pack via packfile.UpdateObjectStorage, 39 + // which has two paths: a storer.PackfileWriter fast-path that copies the client 40 + // stream to EOF, and a parser fallback that reads exactly one self-delimiting 41 + // pack. A git:// client keeps the socket open after its pack, waiting for 42 + // report-status, so it never sends EOF and the fast-path copy deadlocks. By not 43 + // advertising PackfileWriter we take the parser path, which stops at the pack 44 + // trailer. Objects land as loose objects rather than a packfile. 45 + type plainStorer struct { 46 + storage.Storer 47 + } 48 + 49 + // Serve accepts connections on l until ctx is cancelled or Accept fails. 50 + func (d *daemon) Serve(ctx context.Context, l net.Listener) error { 51 + go func() { 52 + <-ctx.Done() 53 + _ = l.Close() 54 + }() 55 + 56 + for { 57 + conn, err := l.Accept() 58 + if err != nil { 59 + if ctx.Err() != nil { 60 + return nil 61 + } 62 + return fmt.Errorf("objgitd: accept: %w", err) 63 + } 64 + 65 + go func() { 66 + if err := d.handle(ctx, conn); err != nil { 67 + slog.Error("connection failed", 68 + "remote", conn.RemoteAddr().String(), 69 + "err", err, 70 + ) 71 + } 72 + }() 73 + } 74 + } 75 + 76 + // handle services a single git:// connection: decode the request line, resolve 77 + // the repository, and hand the socket to the matching server command. 78 + func (d *daemon) handle(ctx context.Context, conn net.Conn) error { 79 + defer conn.Close() 80 + 81 + // A silent client must not be able to pin a goroutine forever. 82 + _ = conn.SetReadDeadline(time.Now().Add(handshakeTimeout)) 83 + 84 + var req packp.GitProtoRequest 85 + if err := req.Decode(conn); err != nil { 86 + return fmt.Errorf("decoding git-proto-request: %w", err) 87 + } 88 + 89 + // The transfer that follows can take a while; drop the handshake deadline. 90 + _ = conn.SetReadDeadline(time.Time{}) 91 + 92 + slog.Info("serving request", 93 + "service", req.RequestCommand, 94 + "path", req.Pathname, 95 + "remote", conn.RemoteAddr().String(), 96 + ) 97 + 98 + // ExtraParams carries e.g. "version=2"; transport.ProtocolVersion splits on ":". 99 + gitProtocol := strings.Join(req.ExtraParams, ":") 100 + 101 + // UploadPack/ReceivePack call r.Close() between negotiation rounds, so the 102 + // reader must be a no-op closer or the socket dies mid-conversation. The 103 + // writer is the raw conn: its final Close() ends the connection. 104 + r := io.NopCloser(conn) 105 + 106 + switch req.RequestCommand { 107 + case transport.UploadPackService: 108 + st, err := d.loader.Load(&url.URL{Path: req.Pathname}) 109 + if err != nil { 110 + _, _ = pktline.WriteError(conn, fmt.Errorf("repository %q not found", req.Pathname)) 111 + return fmt.Errorf("loading %q: %w", req.Pathname, err) 112 + } 113 + return transport.UploadPack(ctx, st, r, conn, &transport.UploadPackRequest{ 114 + GitProtocol: gitProtocol, 115 + }) 116 + 117 + case transport.ReceivePackService: 118 + if !d.allowPush { 119 + _, _ = pktline.WriteError(conn, fmt.Errorf("push is disabled on this server")) 120 + return fmt.Errorf("push rejected for %q", req.Pathname) 121 + } 122 + st, err := d.loadOrInit(req.Pathname) 123 + if err != nil { 124 + _, _ = pktline.WriteError(conn, fmt.Errorf("cannot open repository %q", req.Pathname)) 125 + return fmt.Errorf("opening %q for push: %w", req.Pathname, err) 126 + } 127 + return transport.ReceivePack(ctx, plainStorer{st}, r, conn, &transport.ReceivePackRequest{ 128 + GitProtocol: gitProtocol, 129 + }) 130 + 131 + default: 132 + _, _ = pktline.WriteError(conn, fmt.Errorf("unsupported service %q", req.RequestCommand)) 133 + return fmt.Errorf("unsupported service: %s", req.RequestCommand) 134 + } 135 + } 136 + 137 + // loadOrInit returns the storer for repoPath, creating an empty bare repository 138 + // on demand. Git's daemon never auto-creates; objgitd does, so a first push to 139 + // a new path just works. 140 + func (d *daemon) loadOrInit(repoPath string) (storage.Storer, error) { 141 + st, err := d.loader.Load(&url.URL{Path: repoPath}) 142 + if err == nil { 143 + return st, nil 144 + } 145 + if !errors.Is(err, transport.ErrRepositoryNotFound) { 146 + return nil, err 147 + } 148 + 149 + fs, err := d.fs.Chroot(repoPath) 150 + if err != nil { 151 + return nil, fmt.Errorf("chroot %q: %w", repoPath, err) 152 + } 153 + 154 + st = filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) 155 + if _, err := git.Init(st, git.WithDefaultBranch(plumbing.NewBranchReferenceName("main"))); err != nil { 156 + return nil, fmt.Errorf("init bare repo: %w", err) 157 + } 158 + 159 + slog.Info("created repository", "path", repoPath) 160 + return st, nil 161 + }
+134
cmd/objgitd/daemon_test.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "net" 6 + "os" 7 + "os/exec" 8 + "path/filepath" 9 + "strings" 10 + "testing" 11 + "time" 12 + 13 + "github.com/go-git/go-billy/v6/memfs" 14 + "github.com/go-git/go-git/v6/plumbing/transport" 15 + ) 16 + 17 + // TestDaemonPushCreatesRepo reproduces "git push git://host/new.git" against a 18 + // path that does not exist yet. The daemon must create the bare repository on 19 + // demand and the result must clone back cleanly. 20 + func TestDaemonPushCreatesRepo(t *testing.T) { 21 + if _, err := exec.LookPath("git"); err != nil { 22 + t.Skip("git not installed") 23 + } 24 + 25 + fs := memfs.New() 26 + d := &daemon{ 27 + fs: fs, 28 + loader: transport.NewFilesystemLoader(fs, false), 29 + allowPush: true, 30 + } 31 + 32 + ctx, cancel := context.WithCancel(context.Background()) 33 + defer cancel() 34 + 35 + ln, err := net.Listen("tcp", "127.0.0.1:0") 36 + if err != nil { 37 + t.Fatalf("listen: %v", err) 38 + } 39 + 40 + srvErr := make(chan error, 1) 41 + go func() { srvErr <- d.Serve(ctx, ln) }() 42 + 43 + remote := "git://" + ln.Addr().String() + "/test.git" 44 + 45 + work := t.TempDir() 46 + runGit(t, work, "init", "-b", "main") 47 + runGit(t, work, "config", "user.email", "test@example.com") 48 + runGit(t, work, "config", "user.name", "Test") 49 + runGit(t, work, "commit", "--allow-empty", "-m", "initial") 50 + 51 + // The repository does not exist yet; the push must create it. 52 + runGit(t, work, "push", remote, "main") 53 + 54 + if _, err := fs.Stat("/test.git/config"); err != nil { 55 + t.Fatalf("expected bare repo to be created on push, but %q is missing: %v", "/test.git/config", err) 56 + } 57 + 58 + // Round-trip: a clone must recover the pushed commit. 59 + dst := t.TempDir() 60 + runGit(t, dst, "clone", remote, "cloned") 61 + head := strings.TrimSpace(runGit(t, filepath.Join(dst, "cloned"), "rev-parse", "HEAD")) 62 + if head == "" { 63 + t.Fatal("cloned repository has no HEAD") 64 + } 65 + 66 + cancel() 67 + _ = ln.Close() 68 + select { 69 + case <-srvErr: 70 + case <-time.After(2 * time.Second): 71 + t.Fatal("daemon did not shut down") 72 + } 73 + } 74 + 75 + // TestDaemonPushDisabled confirms that pushes are rejected (not silently 76 + // creating repos) when allowPush is false. 77 + func TestDaemonPushDisabled(t *testing.T) { 78 + if _, err := exec.LookPath("git"); err != nil { 79 + t.Skip("git not installed") 80 + } 81 + 82 + fs := memfs.New() 83 + d := &daemon{ 84 + fs: fs, 85 + loader: transport.NewFilesystemLoader(fs, false), 86 + allowPush: false, 87 + } 88 + 89 + ctx, cancel := context.WithCancel(context.Background()) 90 + defer cancel() 91 + 92 + ln, err := net.Listen("tcp", "127.0.0.1:0") 93 + if err != nil { 94 + t.Fatalf("listen: %v", err) 95 + } 96 + go func() { _ = d.Serve(ctx, ln) }() 97 + 98 + remote := "git://" + ln.Addr().String() + "/test.git" 99 + 100 + work := t.TempDir() 101 + runGit(t, work, "init", "-b", "main") 102 + runGit(t, work, "config", "user.email", "test@example.com") 103 + runGit(t, work, "config", "user.name", "Test") 104 + runGit(t, work, "commit", "--allow-empty", "-m", "initial") 105 + 106 + if out, err := tryGit(work, "push", remote, "main"); err == nil { 107 + t.Fatalf("expected push to be rejected when allowPush is false, got success:\n%s", out) 108 + } 109 + 110 + if _, err := fs.Stat("/test.git/config"); err == nil { 111 + t.Fatal("repository must not be created when push is disabled") 112 + } 113 + } 114 + 115 + func runGit(t *testing.T, dir string, args ...string) string { 116 + t.Helper() 117 + out, err := tryGit(dir, args...) 118 + if err != nil { 119 + t.Fatalf("git %v: %v\n%s", args, err, out) 120 + } 121 + return out 122 + } 123 + 124 + func tryGit(dir string, args ...string) (string, error) { 125 + cmd := exec.Command("git", args...) 126 + cmd.Dir = dir 127 + cmd.Env = append(os.Environ(), 128 + "GIT_CONFIG_GLOBAL=/dev/null", 129 + "GIT_CONFIG_SYSTEM=/dev/null", 130 + "GIT_TERMINAL_PROMPT=0", 131 + ) 132 + out, err := cmd.CombinedOutput() 133 + return string(out), err 134 + }
+68
cmd/objgitd/main.go
··· 1 1 package main 2 2 3 3 import ( 4 + "context" 4 5 "flag" 6 + "log/slog" 7 + "net" 8 + "os" 9 + "os/signal" 10 + "syscall" 5 11 6 12 "github.com/facebookgo/flagenv" 13 + "github.com/go-git/go-git/v6/plumbing/transport" 14 + "github.com/tigrisdata/storage-go" 15 + "tangled.org/xeiaso.net/objgit/internal/s3fs" 16 + 17 + _ "github.com/joho/godotenv/autoload" 18 + ) 19 + 20 + var ( 21 + bind = flag.String("bind", ":9418", "TCP address to listen on for the git:// protocol") 22 + bucket = flag.String("bucket", "", "Tigris bucket that holds the git repositories") 23 + allowPush = flag.Bool("allow-push", false, "allow unauthenticated git-receive-pack (push) requests") 24 + slogLevel = flag.String("slog-level", "INFO", "log level (DEBUG, INFO, WARN, ERROR)") 7 25 ) 8 26 9 27 func main() { 10 28 flagenv.Parse() 11 29 flag.Parse() 30 + 31 + var lvl slog.Level 32 + if err := lvl.UnmarshalText([]byte(*slogLevel)); err != nil { 33 + slog.Error("invalid -slog-level", "value", *slogLevel, "err", err) 34 + os.Exit(1) 35 + } 36 + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: lvl}))) 37 + 38 + if *bucket == "" { 39 + slog.Error("-bucket is required") 40 + os.Exit(1) 41 + } 42 + 43 + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) 44 + defer cancel() 45 + 46 + client, err := storage.New(ctx) 47 + if err != nil { 48 + slog.Error("can't create Tigris storage client", "err", err) 49 + os.Exit(1) 50 + } 51 + 52 + fsys, err := s3fs.NewS3FS(client, *bucket) 53 + if err != nil { 54 + slog.Error("can't create s3fs", "bucket", *bucket, "err", err) 55 + os.Exit(1) 56 + } 57 + 58 + d := &daemon{ 59 + fs: fsys, 60 + loader: transport.NewFilesystemLoader(fsys, false), 61 + allowPush: *allowPush, 62 + } 63 + 64 + ln, err := net.Listen("tcp", *bind) 65 + if err != nil { 66 + slog.Error("can't listen", "bind", *bind, "err", err) 67 + os.Exit(1) 68 + } 69 + 70 + slog.Info("objgitd listening", 71 + "bind", *bind, 72 + "bucket", *bucket, 73 + "allow_push", *allowPush, 74 + ) 75 + 76 + if err := d.Serve(ctx, ln); err != nil { 77 + slog.Error("server stopped", "err", err) 78 + os.Exit(1) 79 + } 12 80 }
+15
go.mod
··· 8 8 github.com/aws/smithy-go v1.26.0 9 9 github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 10 10 github.com/go-git/go-billy/v6 v6.0.0-alpha.1 11 + github.com/go-git/go-git/v6 v6.0.0-alpha.4 12 + github.com/joho/godotenv v1.5.1 11 13 github.com/tigrisdata/storage-go v0.6.0 12 14 go.uber.org/atomic v1.11.0 13 15 ) 14 16 15 17 require ( 18 + github.com/Microsoft/go-winio v0.6.2 // indirect 19 + github.com/ProtonMail/go-crypto v1.4.1 // indirect 16 20 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect 17 21 github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect 18 22 github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect ··· 29 33 github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect 30 34 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect 31 35 github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect 36 + github.com/cloudflare/circl v1.6.3 // indirect 37 + github.com/emirpasic/gods v1.18.1 // indirect 32 38 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect 33 39 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect 34 40 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect 41 + github.com/go-git/gcfg/v2 v2.0.2 // indirect 42 + github.com/kevinburke/ssh_config v1.6.0 // indirect 43 + github.com/klauspost/cpuid/v2 v2.3.0 // indirect 44 + github.com/pjbgf/sha1cd v0.6.0 // indirect 45 + github.com/sergi/go-diff v1.4.0 // indirect 46 + golang.org/x/crypto v0.51.0 // indirect 47 + golang.org/x/net v0.54.0 // indirect 48 + golang.org/x/sync v0.20.0 // indirect 49 + golang.org/x/sys v0.44.0 // indirect 35 50 )
+50
go.sum
··· 1 + github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 2 + github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 3 + github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= 4 + github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= 5 + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= 6 + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= 7 + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= 8 + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 1 9 github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= 2 10 github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= 3 11 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= ··· 36 44 github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= 37 45 github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s= 38 46 github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= 47 + github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= 48 + github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= 49 + github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 39 50 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 40 51 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 52 + github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= 53 + github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= 41 54 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= 42 55 github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= 43 56 github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 h1:CkmB2l68uhvRlwOTPrwnuitSxi/S3Cg4L5QYOcL9MBc= ··· 46 59 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= 47 60 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= 48 61 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= 62 + github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= 63 + github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= 64 + github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo= 65 + github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs= 49 66 github.com/go-git/go-billy/v6 v6.0.0-alpha.1 h1:xVjAR4oUvrKy7/Xuw/lLlV3gkxR3KO2H8W+MamuVVsQ= 50 67 github.com/go-git/go-billy/v6 v6.0.0-alpha.1/go.mod h1:eaCUpHbedW7//EwcYmUDfJe2N6sJC9O12AT0OTqJR1E= 68 + github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA= 69 + github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ= 70 + github.com/go-git/go-git/v6 v6.0.0-alpha.4 h1:aDTc2UGanmaE7FkGLSlBEB9nohMnQ+RKXcfq/D+esDQ= 71 + github.com/go-git/go-git/v6 v6.0.0-alpha.4/go.mod h1:4ODa/G7hPWrh4Y+7lmt59Ij3zW38IEfvRoAZxLYYBhc= 51 72 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 52 73 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 74 + github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= 75 + github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= 76 + github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= 77 + github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 78 + github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 79 + github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 80 + github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 81 + github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= 82 + github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= 53 83 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 54 84 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 85 + github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= 86 + github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= 87 + github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 88 + github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 55 89 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 56 90 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 57 91 github.com/tigrisdata/storage-go v0.6.0 h1:1Rxzxp/GAUqwFln8qMXYOQIa47IsT2aBNuxYc55o6D4= 58 92 github.com/tigrisdata/storage-go v0.6.0/go.mod h1:l3u7N9LDIhv4lfpkEBJYzolWJ/SBb6WiBexgy/uq6iQ= 59 93 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 60 94 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 95 + golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= 96 + golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= 97 + golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= 98 + golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= 99 + golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= 100 + golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= 101 + golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= 102 + golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= 103 + golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= 104 + golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= 105 + golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= 106 + golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= 107 + gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 108 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 109 + gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 110 + gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 61 111 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 112 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+20 -32
internal/s3fs/basic.go
··· 54 54 return nil, errors.New("unsupported open flag") 55 55 } 56 56 57 - // Get the file path 58 - p := path.Join(fs3.root, filename) 57 + // Canonical S3 key for this path. Every branch uses it so reads and 58 + // writes resolve to the same object regardless of chroot depth. 59 + key := fs3.key(filename) 59 60 60 61 switch flag & SupportedOFlags { 61 62 case O_RDONLY: 62 63 // The bucket root is always a directory; short-circuit so WASI 63 64 // preopens (which OpenFile(".", O_RDONLY)) don't issue an S3 call. 64 - key := strings.TrimPrefix(fs3.cleanPath(filename), "/") 65 65 if key == "" || key == "." { 66 - return newS3DirFile(p, fs3.bucket, fs3.client), nil 66 + return newS3DirFile(key, fs3.bucket, fs3.client), nil 67 67 } 68 68 69 - f, err := newS3ReadFile(fs3.client, fs3.bucket, p) 69 + f, err := newS3ReadFile(fs3.client, fs3.bucket, key, filename) 70 70 if err == nil { 71 71 return f, nil 72 72 } ··· 96 96 return nil, lerr 97 97 } 98 98 if len(list.Contents) > 0 || len(list.CommonPrefixes) > 0 { 99 - return newS3DirFile(p, fs3.bucket, fs3.client), nil 99 + return newS3DirFile(key, fs3.bucket, fs3.client), nil 100 100 } 101 101 return nil, &os.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist} 102 102 103 103 case O_WRONLY: 104 - return newS3WriteFile(fs3.client, fs3.bucket, p) 104 + return newS3WriteFile(fs3.client, fs3.bucket, key, filename) 105 105 106 106 case O_WRMULTIPART: 107 - return newS3MultipartUploadFile(fs3.client, fs3.bucket, p) 107 + return newS3MultipartUploadFile(fs3.client, fs3.bucket, key, filename) 108 108 109 109 default: 110 110 return nil, errors.New("unsupported open flag") ··· 164 164 // is not a directory, Rename replaces it. OS-specific restrictions may 165 165 // apply when oldpath and newpath are in different directories. 166 166 func (fs3 *S3FS) Rename(oldpath, newpath string) error { 167 - // TODO: Validate the paths? 168 - 169 - // Create a context 170 167 ctx := context.TODO() // TODO: Get user-supplied context? 171 168 172 - // Format the paths 173 - src := path.Join(fs3.root, oldpath) 174 - dst := path.Join(fs3.root, newpath) 169 + src := fs3.key(oldpath) 170 + dst := fs3.key(newpath) 175 171 176 - // Send the copy request 177 - _, err := fs3.client.CopyObject(ctx, &s3.CopyObjectInput{ 172 + // RenameObject is a Tigris extension that renames in place (no data copy), 173 + // so we don't need a separate CopyObject + DeleteObject. CopySource is 174 + // bucket-qualified; Key is the destination key. 175 + copySource := fs3.bucket + "/" + src 176 + _, err := fs3.client.RenameObject(ctx, &s3.CopyObjectInput{ 178 177 Bucket: &fs3.bucket, 179 - CopySource: &src, 178 + CopySource: &copySource, 180 179 Key: &dst, 181 180 }) 182 181 if err != nil { 183 - return fmt.Errorf("failed to rename file: %s", err) 184 - } 185 - 186 - // Delete the old file 187 - // TODO: Parse the response? 188 - _, err = fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 189 - Bucket: &fs3.bucket, 190 - Key: &src, 191 - }) 192 - if err != nil { 193 - return fmt.Errorf("failed to remove file: %s", err) 182 + return fmt.Errorf("failed to rename %q to %q: %w", oldpath, newpath, err) 194 183 } 195 184 196 185 return nil ··· 204 193 // Create a context 205 194 ctx := context.TODO() // TODO: Get user-supplied context? 206 195 207 - // Format the path 208 - p := path.Join(fs3.root, filename) 196 + key := fs3.key(filename) 209 197 210 198 // Send the request 211 199 // TODO: Parse the response? 212 200 _, err := fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 213 201 Bucket: &fs3.bucket, 214 - Key: &p, 202 + Key: &key, 215 203 }) 216 204 if err != nil { 217 - return fmt.Errorf("failed to remove file: %s", err) 205 + return fmt.Errorf("failed to remove file: %w", err) 218 206 } 219 207 return nil 220 208 }
+8 -4
internal/s3fs/chroot.go
··· 14 14 // Calculate the new root 15 15 p := fs3.Join(fs3.root, path) 16 16 17 - // Create the new S3FS with the new root directory 17 + // Create the new S3FS with the new root directory. The separator must be 18 + // carried over; without it ListObjectsV2 runs with an empty delimiter and 19 + // ReadDir flattens the tree, breaking directory-structured reads (e.g. git 20 + // ref enumeration under refs/). 18 21 nfs := &S3FS{ 19 - client: fs3.client, 20 - bucket: fs3.bucket, 21 - root: p, 22 + client: fs3.client, 23 + bucket: fs3.bucket, 24 + root: p, 25 + separator: fs3.separator, 22 26 } 23 27 return nfs, nil 24 28 }
+23
internal/s3fs/chroot_test.go
··· 1 + package s3fs 2 + 3 + import "testing" 4 + 5 + // TestChrootPreservesSeparator guards against dropping the separator when 6 + // chrooting. An empty separator makes ListObjectsV2 run without a delimiter, 7 + // which flattens ReadDir and breaks directory-structured reads such as git ref 8 + // enumeration under refs/. 9 + func TestChrootPreservesSeparator(t *testing.T) { 10 + base := &S3FS{bucket: "bucket", separator: DefaultSeparator} 11 + 12 + sub, err := base.Chroot("repo.git") 13 + if err != nil { 14 + t.Fatalf("Chroot: %v", err) 15 + } 16 + 17 + got := sub.(*S3FS).separator 18 + if got != DefaultSeparator { 19 + t.Logf("want: %q", DefaultSeparator) 20 + t.Logf("got: %q", got) 21 + t.Error("Chroot dropped the separator") 22 + } 23 + }
+8 -2
internal/s3fs/dir.go
··· 3 3 package s3fs 4 4 5 5 import ( 6 + "bytes" 6 7 "context" 7 - "errors" 8 8 "io/fs" 9 9 "os" 10 10 pathpkg "path" ··· 80 80 // perm are used for all directories that MkdirAll creates. If path is/ 81 81 // already a directory, MkdirAll does nothing and returns nil. 82 82 func (fs3 *S3FS) MkdirAll(filename string, perm os.FileMode) error { 83 - return errors.New("not implemented") 83 + _, err := fs3.client.PutObject(context.TODO(), &s3.PutObjectInput{ 84 + Bucket: new(fs3.bucket), 85 + Key: new(filename), 86 + Body: bytes.NewBuffer(nil), 87 + }) 88 + 89 + return err 84 90 }
+20 -14
internal/s3fs/file.go
··· 37 37 client *storage.Client // S3 SDK client 38 38 bucket string // S3 bucket name 39 39 key string // File object's key in S3 40 + name string // Root-relative path as presented to Open 40 41 closed bool // Is the file closed? 41 42 reader *bytes.Reader // Buffer for file contents 42 43 head *s3.HeadObjectOutput // File metadata from S3 43 44 } 44 45 45 - // newS3ReadFile creates a new s3ReadFile. 46 - func newS3ReadFile(client *storage.Client, bucket, key string) (*s3ReadFile, error) { 46 + // newS3ReadFile creates a new s3ReadFile. key is the full S3 object key; name 47 + // is the root-relative path the caller passed to Open (returned by Name). 48 + func newS3ReadFile(client *storage.Client, bucket, key, name string) (*s3ReadFile, error) { 47 49 // Create the context 48 50 ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 49 51 ··· 76 78 client: client, 77 79 bucket: bucket, 78 80 key: key, 81 + name: name, 79 82 reader: reader, 80 83 head: ho, 81 84 }, nil ··· 83 86 84 87 // Name returns the name of the file as presented to Open. 85 88 func (f *s3ReadFile) Name() string { 86 - return f.key 89 + return f.name 87 90 } 88 91 89 92 // Write implements io.Writer for billy.File ··· 159 162 client *storage.Client // s3 skd client 160 163 bucket string // S3 bucket name 161 164 key string // File object's key in S3 165 + name string // Root-relative path as presented to Open 162 166 closed bool // Is the file closed? 163 167 buf *bytes.Buffer // Buffer for storing the file before it's uploaded 164 168 } 165 169 166 - // newS3WriteFile creates a new s3ReadFile. 167 - func newS3WriteFile(client *storage.Client, bucket, key string) (*s3WriteFile, error) { 168 - // TODO: Validate the key 169 - // ... 170 - 170 + // newS3WriteFile creates a new s3WriteFile. key is the full S3 object key; name 171 + // is the root-relative path the caller passed to Open (returned by Name). 172 + func newS3WriteFile(client *storage.Client, bucket, key, name string) (*s3WriteFile, error) { 171 173 return &s3WriteFile{ 172 174 client: client, 173 175 bucket: bucket, 174 176 key: key, 177 + name: name, 175 178 buf: bytes.NewBuffer(nil), 176 179 }, nil 177 180 } 178 181 179 182 // Name returns the name of the file as presented to Open. 180 183 func (f *s3WriteFile) Name() string { 181 - return f.key 184 + return f.name 182 185 } 183 186 184 187 // Write implements os.Writer for billy.File ··· 205 208 206 209 // Seek implements io.Seeker for billy.File 207 210 func (f *s3WriteFile) Seek(offset int64, whence int) (int64, error) { 208 - return 0, ErrNotImplemented 211 + return 0, &os.PathError{Op: "seek", Path: f.key, Err: ErrNotImplemented} 209 212 } 210 213 211 214 // Close implements io.Closer for billy.File ··· 262 265 client *storage.Client // s3 skd client 263 266 bucket string // S3 bucket name 264 267 key string // File object's key in S3 268 + name string // Root-relative path as presented to Open 265 269 closed bool // Is the file closed? 266 270 uploadID string // S3 multipart upload ID 267 271 uploadN *atomic.Int32 // Counter tracking the number of uploads 268 272 } 269 273 270 - // newS3MultipartUploadFile creates a new s3ReadFile. 271 - func newS3MultipartUploadFile(client *storage.Client, bucket, key string) (*s3MultipartUploadFile, error) { 274 + // newS3MultipartUploadFile creates a new s3MultipartUploadFile. key is the full 275 + // S3 object key; name is the root-relative path passed to Open. 276 + func newS3MultipartUploadFile(client *storage.Client, bucket, key, name string) (*s3MultipartUploadFile, error) { 272 277 // TODO: Check if the file exists 273 278 // ... 274 279 ··· 289 294 client: client, 290 295 bucket: bucket, 291 296 key: key, 297 + name: name, 292 298 uploadID: *res.UploadId, 293 299 uploadN: atomic.NewInt32(1), 294 300 }, nil 295 301 } 296 302 297 303 // Name returns the name of the file as presented to Open. 298 - func (f *s3MultipartUploadFile) Name() string { return f.key } 304 + func (f *s3MultipartUploadFile) Name() string { return f.name } 299 305 300 306 // Write implements os.Writer for billy.File 301 307 func (f *s3MultipartUploadFile) Write(p []byte) (n int, err error) { ··· 346 352 347 353 // Seek implements io.Seeker for billy.File 348 354 func (f *s3MultipartUploadFile) Seek(offset int64, whence int) (int64, error) { 349 - return 0, errors.New("seek not implemented") 355 + return 0, &os.PathError{Op: "seek", Path: f.key, Err: ErrNotImplemented} 350 356 } 351 357 352 358 // Close implements io.Closer for billy.File
+9
internal/s3fs/filesystem.go
··· 3 3 import ( 4 4 "fmt" 5 5 "path" 6 + "strings" 6 7 7 8 "github.com/go-git/go-billy/v6" 8 9 "github.com/tigrisdata/storage-go" ··· 51 52 // Return the full path 52 53 return path.Clean(f) 53 54 } 55 + 56 + // key turns a root-relative billy path into the canonical S3 object key: 57 + // root-joined, cleaned, and stripped of the leading slash that S3 keys never 58 + // carry. All S3 operations must funnel through here so reads and writes agree 59 + // on the same key regardless of chroot depth. 60 + func (fs3 *S3FS) key(name string) string { 61 + return strings.TrimPrefix(fs3.cleanPath(name), "/") 62 + }
+61
internal/s3fs/key_test.go
··· 1 + package s3fs 2 + 3 + import ( 4 + "strings" 5 + "testing" 6 + ) 7 + 8 + func chrootedFS(t *testing.T, root string) *S3FS { 9 + t.Helper() 10 + sub, err := (&S3FS{separator: DefaultSeparator}).Chroot(root) 11 + if err != nil { 12 + t.Fatalf("Chroot(%q): %v", root, err) 13 + } 14 + return sub.(*S3FS) 15 + } 16 + 17 + // TestKeyRootedAndSlashFree pins the invariant the chroot fix depends on: an 18 + // S3 key is the root-joined path with no leading slash, so writes (OpenFile) 19 + // and lookups (Stat/Rename) resolve to the same object. 20 + func TestKeyRootedAndSlashFree(t *testing.T) { 21 + fs := chrootedFS(t, "/xeiaso.net/objgit.git") 22 + 23 + for _, tt := range []struct { 24 + name, in, want string 25 + }{ 26 + {name: "plain", in: "config", want: "xeiaso.net/objgit.git/config"}, 27 + {name: "nested", in: "objects/ab/cd", want: "xeiaso.net/objgit.git/objects/ab/cd"}, 28 + {name: "leading slash input", in: "/HEAD", want: "xeiaso.net/objgit.git/HEAD"}, 29 + } { 30 + t.Run(tt.name, func(t *testing.T) { 31 + got := fs.key(tt.in) 32 + if got != tt.want { 33 + t.Logf("want: %q", tt.want) 34 + t.Logf("got: %q", got) 35 + t.Error("wrong S3 key") 36 + } 37 + }) 38 + } 39 + } 40 + 41 + // TestTempFileNameIsRootRelative guards the contract go-git's dotgit relies on: 42 + // File.Name() is root-relative, so fs.Rename(file.Name(), dst) works. The S3 43 + // key derived from that name must still carry the chroot prefix. 44 + func TestTempFileNameIsRootRelative(t *testing.T) { 45 + fs := chrootedFS(t, "/repo.git") 46 + 47 + f, err := fs.TempFile("objects/pack", "tmp_obj_") 48 + if err != nil { 49 + t.Fatalf("TempFile: %v", err) 50 + } 51 + 52 + name := f.Name() 53 + if !strings.HasPrefix(name, "objects/pack/tmp_obj_") { 54 + t.Fatalf("Name() = %q, want a root-relative path under objects/pack/tmp_obj_", name) 55 + } 56 + 57 + const wantPrefix = "repo.git/objects/pack/tmp_obj_" 58 + if got := fs.key(name); !strings.HasPrefix(got, wantPrefix) { 59 + t.Fatalf("key(Name()) = %q, want prefix %q", got, wantPrefix) 60 + } 61 + }
+22 -10
internal/s3fs/tempfile.go
··· 2 2 3 3 package s3fs 4 4 5 - import "github.com/go-git/go-billy/v6" 5 + import ( 6 + "crypto/rand" 7 + "encoding/hex" 8 + "fmt" 6 9 7 - // TempFile creates a new temporary file in the directory dir with a name 8 - // beginning with prefix, opens the file for reading and writing, and 9 - // returns the resulting *os.File. If dir is the empty string, TempFile 10 - // uses the default directory for temporary files (see os.TempDir). 11 - // Multiple programs calling TempFile simultaneously will not choose the 12 - // same file. The caller can use f.Name() to find the pathname of the file. 13 - // It is the caller's responsibility to remove the file when no longer 14 - // needed. 10 + "github.com/go-git/go-billy/v6" 11 + ) 12 + 13 + // TempFile creates a uniquely named, write-only file under dir whose name 14 + // begins with prefix. The object is uploaded to S3 when the returned file is 15 + // closed; until then nothing exists in the bucket. The caller is responsible 16 + // for renaming or removing it. 17 + // 18 + // Note: the returned file is write-only. S3 has no read-while-write temp file, 19 + // so callers that reopen the temp path for reading before Close (e.g. go-git's 20 + // streaming PackWriter) are not supported; use the loose-object path instead. 15 21 func (fs3 *S3FS) TempFile(dir, prefix string) (billy.File, error) { 16 - return nil, nil 22 + var b [16]byte 23 + if _, err := rand.Read(b[:]); err != nil { 24 + return nil, fmt.Errorf("s3fs: generating temp file name: %w", err) 25 + } 26 + 27 + name := fs3.Join(dir, prefix+hex.EncodeToString(b[:])) 28 + return newS3WriteFile(fs3.client, fs3.bucket, fs3.key(name), name) 17 29 }