Git backed by object storage because you can't stop me
git
object-storage
kefka
fix(s3fs): harden S3 client transport to fail fast on stale connections
The AWS SDK's default transport (IdleConnTimeout: 90s, no
ResponseHeaderTimeout) allows reused keep-alive connections to Tigris
to sit stale and never respond. With context.TODO() (no timeout),
ReadObject hangs forever instead of failing and retrying.
Inject a hardened HTTP client (ResponseHeaderTimeout: 30s,
IdleConnTimeout: 30s) into every S3 call via per-call options, so
stale connections error quickly and the retryer recovers.
Relates to: lazy-read clone hangs (independently useful).
Assisted-by: Claude Opus 4.8 via claude.ai
Signed-off-by: Xe Iaso <me@xeiaso.net>
+255
+109
internal/s3fs/resilient.go
+109
internal/s3fs/resilient.go
···
1
+
package s3fs
2
+
3
+
import (
4
+
"context"
5
+
"net/http"
6
+
"time"
7
+
8
+
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
9
+
"github.com/aws/aws-sdk-go-v2/service/s3"
10
+
)
11
+
12
+
// Stale keep-alive connections are the failure this guards against. The S3
13
+
// client storage-go builds uses the AWS SDK default transport: IdleConnTimeout
14
+
// 90s and — unlike storage-go's own bundle client — no ResponseHeaderTimeout.
15
+
// Tigris (or an intermediary/NAT) silently drops idle connections well before
16
+
// 90s, so a pooled connection can be dead while the client still believes it is
17
+
// usable. A request written to such a connection is never answered; with no
18
+
// response-header deadline and the call's context.TODO() carrying no timeout,
19
+
// the read blocks forever.
20
+
//
21
+
// The lazy reader (see file.go) made this acute: where the old eager reader did
22
+
// one GetObject per pack file in a burst on fresh connections, the lazy reader
23
+
// issues many small GetObjects strung out across go-git's processing — long
24
+
// enough for pooled connections to go stale before reuse. A single clone of a
25
+
// real repository reliably wedges.
26
+
//
27
+
// hardenedTimeouts bound those two windows: prune idle connections before
28
+
// Tigris does (so they are never reused stale), and fail a stalled reused
29
+
// connection fast (so the SDK retryer retries the idempotent request on a fresh
30
+
// connection instead of hanging). ResponseHeaderTimeout bounds only the wait
31
+
// for response headers, not body streaming, so large pack reads are unaffected.
32
+
const (
33
+
hardenedIdleConnTimeout = 30 * time.Second
34
+
hardenedResponseHeaderTimeout = 30 * time.Second
35
+
)
36
+
37
+
// newHardenedHTTPClient returns a single AWS HTTP client whose transport prunes
38
+
// idle connections early and times out the wait for response headers. It must
39
+
// be shared across all calls so its connection pool is reused; a per-call client
40
+
// would defeat pooling.
41
+
func newHardenedHTTPClient() *awshttp.BuildableClient {
42
+
return awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
43
+
t.IdleConnTimeout = hardenedIdleConnTimeout
44
+
t.ResponseHeaderTimeout = hardenedResponseHeaderTimeout
45
+
})
46
+
}
47
+
48
+
// resilientClient wraps an s3Client, injecting a shared hardened HTTP client
49
+
// into every request via a per-call option. The embedded s3Client supplies any
50
+
// method not overridden below; all of them are, so the option reaches every S3
51
+
// round-trip the filesystem makes.
52
+
type resilientClient struct {
53
+
s3Client
54
+
opt func(*s3.Options)
55
+
}
56
+
57
+
// Harden wraps a Tigris/S3 client so every request it makes carries an HTTP
58
+
// client that fails fast on stale keep-alive connections rather than hanging
59
+
// forever. Pass the result to NewS3FS and NewListingCache. See the package
60
+
// constants above for the rationale.
61
+
func Harden(c s3Client) s3Client {
62
+
hc := newHardenedHTTPClient()
63
+
return resilientClient{
64
+
s3Client: c,
65
+
opt: func(o *s3.Options) { o.HTTPClient = hc },
66
+
}
67
+
}
68
+
69
+
// withOpt prepends the hardening option so an explicit per-call option still
70
+
// takes precedence (later options win in the SDK's option application).
71
+
func (c resilientClient) withOpt(opts []func(*s3.Options)) []func(*s3.Options) {
72
+
return append([]func(*s3.Options){c.opt}, opts...)
73
+
}
74
+
75
+
func (c resilientClient) HeadObject(ctx context.Context, in *s3.HeadObjectInput, opts ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
76
+
return c.s3Client.HeadObject(ctx, in, c.withOpt(opts)...)
77
+
}
78
+
79
+
func (c resilientClient) GetObject(ctx context.Context, in *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
80
+
return c.s3Client.GetObject(ctx, in, c.withOpt(opts)...)
81
+
}
82
+
83
+
func (c resilientClient) PutObject(ctx context.Context, in *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
84
+
return c.s3Client.PutObject(ctx, in, c.withOpt(opts)...)
85
+
}
86
+
87
+
func (c resilientClient) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input, opts ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
88
+
return c.s3Client.ListObjectsV2(ctx, in, c.withOpt(opts)...)
89
+
}
90
+
91
+
func (c resilientClient) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput, opts ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
92
+
return c.s3Client.DeleteObject(ctx, in, c.withOpt(opts)...)
93
+
}
94
+
95
+
func (c resilientClient) RenameObject(ctx context.Context, in *s3.CopyObjectInput, opts ...func(*s3.Options)) (*s3.CopyObjectOutput, error) {
96
+
return c.s3Client.RenameObject(ctx, in, c.withOpt(opts)...)
97
+
}
98
+
99
+
func (c resilientClient) CreateMultipartUpload(ctx context.Context, in *s3.CreateMultipartUploadInput, opts ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) {
100
+
return c.s3Client.CreateMultipartUpload(ctx, in, c.withOpt(opts)...)
101
+
}
102
+
103
+
func (c resilientClient) UploadPart(ctx context.Context, in *s3.UploadPartInput, opts ...func(*s3.Options)) (*s3.UploadPartOutput, error) {
104
+
return c.s3Client.UploadPart(ctx, in, c.withOpt(opts)...)
105
+
}
106
+
107
+
func (c resilientClient) CompleteMultipartUpload(ctx context.Context, in *s3.CompleteMultipartUploadInput, opts ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) {
108
+
return c.s3Client.CompleteMultipartUpload(ctx, in, c.withOpt(opts)...)
109
+
}
+146
internal/s3fs/resilient_test.go
+146
internal/s3fs/resilient_test.go
···
1
+
package s3fs
2
+
3
+
import (
4
+
"context"
5
+
"testing"
6
+
7
+
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
8
+
"github.com/aws/aws-sdk-go-v2/service/s3"
9
+
)
10
+
11
+
// optRecorder is an s3Client that records the per-call option functions it
12
+
// receives for each method, so a test can confirm Harden injects its hardened
13
+
// HTTP client on every S3 round-trip. The embedded nil s3Client is never used:
14
+
// the wrapper calls only the methods overridden below.
15
+
type optRecorder struct {
16
+
s3Client
17
+
last []func(*s3.Options)
18
+
}
19
+
20
+
func (r *optRecorder) HeadObject(_ context.Context, _ *s3.HeadObjectInput, opts ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
21
+
r.last = opts
22
+
return &s3.HeadObjectOutput{}, nil
23
+
}
24
+
25
+
func (r *optRecorder) GetObject(_ context.Context, _ *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
26
+
r.last = opts
27
+
return &s3.GetObjectOutput{}, nil
28
+
}
29
+
30
+
func (r *optRecorder) PutObject(_ context.Context, _ *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
31
+
r.last = opts
32
+
return &s3.PutObjectOutput{}, nil
33
+
}
34
+
35
+
func (r *optRecorder) ListObjectsV2(_ context.Context, _ *s3.ListObjectsV2Input, opts ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
36
+
r.last = opts
37
+
return &s3.ListObjectsV2Output{}, nil
38
+
}
39
+
40
+
func (r *optRecorder) DeleteObject(_ context.Context, _ *s3.DeleteObjectInput, opts ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
41
+
r.last = opts
42
+
return &s3.DeleteObjectOutput{}, nil
43
+
}
44
+
45
+
func (r *optRecorder) RenameObject(_ context.Context, _ *s3.CopyObjectInput, opts ...func(*s3.Options)) (*s3.CopyObjectOutput, error) {
46
+
r.last = opts
47
+
return &s3.CopyObjectOutput{}, nil
48
+
}
49
+
50
+
func (r *optRecorder) CreateMultipartUpload(_ context.Context, _ *s3.CreateMultipartUploadInput, opts ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) {
51
+
r.last = opts
52
+
return &s3.CreateMultipartUploadOutput{}, nil
53
+
}
54
+
55
+
func (r *optRecorder) UploadPart(_ context.Context, _ *s3.UploadPartInput, opts ...func(*s3.Options)) (*s3.UploadPartOutput, error) {
56
+
r.last = opts
57
+
return &s3.UploadPartOutput{}, nil
58
+
}
59
+
60
+
func (r *optRecorder) CompleteMultipartUpload(_ context.Context, _ *s3.CompleteMultipartUploadInput, opts ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) {
61
+
r.last = opts
62
+
return &s3.CompleteMultipartUploadOutput{}, nil
63
+
}
64
+
65
+
// TestHardenInjectsHardenedHTTPClient verifies that every method on a Harden-ed
66
+
// client carries the hardened HTTP client (bounded ResponseHeaderTimeout and a
67
+
// reduced IdleConnTimeout) into its per-call options. Without this, S3 requests
68
+
// run on the AWS default transport, whose stale keep-alive connections hang
69
+
// forever — the clone-hang root cause this fix addresses.
70
+
func TestHardenInjectsHardenedHTTPClient(t *testing.T) {
71
+
ctx := context.Background()
72
+
73
+
tests := []struct {
74
+
name string
75
+
invoke func(c s3Client) error
76
+
}{
77
+
{"HeadObject", func(c s3Client) error { _, err := c.HeadObject(ctx, &s3.HeadObjectInput{}); return err }},
78
+
{"GetObject", func(c s3Client) error { _, err := c.GetObject(ctx, &s3.GetObjectInput{}); return err }},
79
+
{"PutObject", func(c s3Client) error { _, err := c.PutObject(ctx, &s3.PutObjectInput{}); return err }},
80
+
{"ListObjectsV2", func(c s3Client) error { _, err := c.ListObjectsV2(ctx, &s3.ListObjectsV2Input{}); return err }},
81
+
{"DeleteObject", func(c s3Client) error { _, err := c.DeleteObject(ctx, &s3.DeleteObjectInput{}); return err }},
82
+
{"RenameObject", func(c s3Client) error { _, err := c.RenameObject(ctx, &s3.CopyObjectInput{}); return err }},
83
+
{"CreateMultipartUpload", func(c s3Client) error {
84
+
_, err := c.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{})
85
+
return err
86
+
}},
87
+
{"UploadPart", func(c s3Client) error { _, err := c.UploadPart(ctx, &s3.UploadPartInput{}); return err }},
88
+
{"CompleteMultipartUpload", func(c s3Client) error {
89
+
_, err := c.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{})
90
+
return err
91
+
}},
92
+
}
93
+
94
+
for _, tt := range tests {
95
+
t.Run(tt.name, func(t *testing.T) {
96
+
rec := &optRecorder{}
97
+
if err := tt.invoke(Harden(rec)); err != nil {
98
+
t.Fatalf("%s: %v", tt.name, err)
99
+
}
100
+
if len(rec.last) == 0 {
101
+
t.Fatalf("%s: no per-call options injected; stale connections would hang", tt.name)
102
+
}
103
+
104
+
// Apply the recorded options the way the SDK would and inspect the
105
+
// resulting HTTP client's transport.
106
+
var o s3.Options
107
+
for _, fn := range rec.last {
108
+
fn(&o)
109
+
}
110
+
if o.HTTPClient == nil {
111
+
t.Fatalf("%s: HTTPClient not set on options", tt.name)
112
+
}
113
+
bc, ok := o.HTTPClient.(*awshttp.BuildableClient)
114
+
if !ok {
115
+
t.Fatalf("%s: HTTPClient is %T, want *awshttp.BuildableClient", tt.name, o.HTTPClient)
116
+
}
117
+
tr := bc.GetTransport()
118
+
if tr.ResponseHeaderTimeout != hardenedResponseHeaderTimeout {
119
+
t.Errorf("%s: ResponseHeaderTimeout = %v, want %v", tt.name, tr.ResponseHeaderTimeout, hardenedResponseHeaderTimeout)
120
+
}
121
+
if tr.IdleConnTimeout != hardenedIdleConnTimeout {
122
+
t.Errorf("%s: IdleConnTimeout = %v, want %v", tt.name, tr.IdleConnTimeout, hardenedIdleConnTimeout)
123
+
}
124
+
})
125
+
}
126
+
}
127
+
128
+
// TestHardenExplicitOptionWins confirms an explicit per-call option still
129
+
// overrides the injected default (later options win), so callers retain control.
130
+
func TestHardenExplicitOptionWins(t *testing.T) {
131
+
rec := &optRecorder{}
132
+
override := awshttp.NewBuildableClient()
133
+
_, err := Harden(rec).GetObject(context.Background(), &s3.GetObjectInput{},
134
+
func(o *s3.Options) { o.HTTPClient = override })
135
+
if err != nil {
136
+
t.Fatal(err)
137
+
}
138
+
139
+
var o s3.Options
140
+
for _, fn := range rec.last {
141
+
fn(&o)
142
+
}
143
+
if got, _ := o.HTTPClient.(*awshttp.BuildableClient); got != override {
144
+
t.Errorf("explicit HTTPClient did not win: got %p, want %p", got, override)
145
+
}
146
+
}