+12

cmd/objgitd/main.go

Reviewed

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "flag" 5 5 + 6 6 + "github.com/facebookgo/flagenv" 7 7 + ) 8 8 + 9 9 + func main() { 10 10 + flagenv.Parse() 11 11 + flag.Parse() 12 12 + }

+35

go.mod

Reviewed

··· 1 1 + module tangled.org/xeiaso.net/objgit 2 2 + 3 3 + go 1.26.3 4 4 + 5 5 + require ( 6 6 + github.com/aws/aws-sdk-go-v2 v1.41.7 7 7 + github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0 8 8 + github.com/aws/smithy-go v1.26.0 9 9 + github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 10 10 + github.com/go-git/go-billy/v6 v6.0.0-alpha.1 11 11 + github.com/tigrisdata/storage-go v0.6.0 12 12 + go.uber.org/atomic v1.11.0 13 13 + ) 14 14 + 15 15 + require ( 16 16 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect 17 17 + github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect 18 18 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect 19 19 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect 20 20 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect 21 21 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect 22 22 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect 23 23 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect 24 24 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect 25 25 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 // indirect 26 26 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect 27 27 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect 28 28 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect 29 29 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect 30 30 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect 31 31 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect 32 32 + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect 33 33 + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect 34 34 + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect 35 35 + )

+62

go.sum

Reviewed

··· 1 1 + github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= 2 2 + github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= 3 3 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= 4 4 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= 5 5 + github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= 6 6 + github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= 7 7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= 8 8 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= 9 9 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= 10 10 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= 11 11 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= 12 12 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= 13 13 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= 14 14 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= 15 15 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= 16 16 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= 17 17 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= 18 18 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= 19 19 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= 20 20 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= 21 21 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16 h1:tX68nPDCoX0s2ksM7CipWP0QFw2hGDWwUdxI6+eT9ZU= 22 22 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.16/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= 23 23 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= 24 24 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= 25 25 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= 26 26 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= 27 27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0 h1:gfPQ6do5PZTCc5n/vZUHz/G8McrNrfERGSO+iHvVbCA= 28 28 + github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0/go.mod h1:wO6U9egJtCtsZEHG2AAcFf1kUWDRrH0Iif6K3bVmmdE= 29 29 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= 30 30 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= 31 31 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= 32 32 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= 33 33 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= 34 34 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= 35 35 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= 36 36 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= 37 37 + github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s= 38 38 + github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= 39 39 + github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 40 40 + github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 41 41 + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= 42 42 + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= 43 43 + github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 h1:CkmB2l68uhvRlwOTPrwnuitSxi/S3Cg4L5QYOcL9MBc= 44 44 + github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456/go.mod h1:zFhibDvPDWmtk4dAQ05sRobtyoffEHygEt3wSNuAzz8= 45 45 + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= 46 46 + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= 47 47 + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= 48 48 + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= 49 49 + github.com/go-git/go-billy/v6 v6.0.0-alpha.1 h1:xVjAR4oUvrKy7/Xuw/lLlV3gkxR3KO2H8W+MamuVVsQ= 50 50 + github.com/go-git/go-billy/v6 v6.0.0-alpha.1/go.mod h1:eaCUpHbedW7//EwcYmUDfJe2N6sJC9O12AT0OTqJR1E= 51 51 + github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 52 52 + github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 53 53 + github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 54 54 + github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 55 55 + github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 56 56 + github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 57 57 + github.com/tigrisdata/storage-go v0.6.0 h1:1Rxzxp/GAUqwFln8qMXYOQIa47IsT2aBNuxYc55o6D4= 58 58 + github.com/tigrisdata/storage-go v0.6.0/go.mod h1:l3u7N9LDIhv4lfpkEBJYzolWJ/SBb6WiBexgy/uq6iQ= 59 59 + go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 60 60 + go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 61 61 + gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 62 62 + gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

+19

internal/s3fs/.gitignore

Reviewed

··· 1 1 + # Binaries for programs and plugins 2 2 + *.exe 3 3 + *.exe~ 4 4 + *.dll 5 5 + *.so 6 6 + *.dylib 7 7 + 8 8 + # Test binary, built with `go test -c` 9 9 + *.test 10 10 + 11 11 + # Output of the go coverage tool, specifically when used with LiteIDE 12 12 + *.out 13 13 + 14 14 + # Dependency directories (remove the comment below to include it) 15 15 + # vendor/ 16 16 + 17 17 + .env 18 18 + .DS_Store 19 19 + tmp/

+21

internal/s3fs/LICENSE

Reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2022 Austin Poor 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+5

internal/s3fs/README.md

Reviewed

··· 1 1 + # s3fs 2 2 + 3 3 + _created by Austin Poor_ 4 4 + 5 5 + An implementation of the [go-billy `Filesystem`](https://pkg.go.dev/github.com/go-git/go-billy/v5#Filesystem) for treating an S3 bucket like a filesystem.

+227

internal/s3fs/basic.go

Reviewed

··· 1 1 + // basic.go implements the interface billy.Basic 2 2 + 3 3 + package s3fs 4 4 + 5 5 + import ( 6 6 + "context" 7 7 + "errors" 8 8 + "fmt" 9 9 + "io/fs" 10 10 + "os" 11 11 + "path" 12 12 + "strings" 13 13 + 14 14 + "github.com/aws/aws-sdk-go-v2/aws" 15 15 + "github.com/aws/aws-sdk-go-v2/service/s3" 16 16 + "github.com/aws/smithy-go" 17 17 + "github.com/go-git/go-billy/v6" 18 18 + ) 19 19 + 20 20 + const ( 21 21 + O_RDONLY int = os.O_RDONLY // open the file read-only. 22 22 + O_WRONLY int = os.O_WRONLY // open the file write-only. 23 23 + O_WRMULTIPART int = 0x4 // open the file for write-only using multipart upload. 24 24 + 25 25 + SupportedOFlags = O_RDONLY | O_WRONLY | O_WRMULTIPART // supported open flags for s3fs 26 26 + ) 27 27 + 28 28 + var ( 29 29 + ErrOpenFlagNotSupported = errors.New("open flag not supported") 30 30 + ) 31 31 + 32 32 + // Create implements billy.Basic 33 33 + // Create creates the named file with mode 0666 (before umask), truncating 34 34 + // it if it already exists. If successful, methods on the returned File can 35 35 + // be used for I/O; the associated file descriptor has mode O_RDWR. 36 36 + func (fs3 *S3FS) Create(filename string) (billy.File, error) { 37 37 + return fs3.OpenFile(filename, O_WRONLY, 0666) 38 38 + } 39 39 + 40 40 + // Open opens the named file for reading. If successful, methods on the 41 41 + // returned file can be used for reading; the associated file descriptor has 42 42 + // mode O_RDONLY. 43 43 + func (fs3 *S3FS) Open(filename string) (billy.File, error) { 44 44 + return fs3.OpenFile(filename, O_RDONLY, 0666) 45 45 + } 46 46 + 47 47 + // OpenFile is the generalized open call; most users will use Open or Create 48 48 + // instead. It opens the named file with specified flag (O_RDONLY etc.) and 49 49 + // perm, (0666 etc.) if applicable. If successful, methods on the returned 50 50 + // File can be used for I/O. 51 51 + func (fs3 *S3FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { 52 52 + // Is the supplied flag supported? 53 53 + if flag&SupportedOFlags != flag { 54 54 + return nil, errors.New("unsupported open flag") 55 55 + } 56 56 + 57 57 + // Get the file path 58 58 + p := path.Join(fs3.root, filename) 59 59 + 60 60 + switch flag & SupportedOFlags { 61 61 + case O_RDONLY: 62 62 + // The bucket root is always a directory; short-circuit so WASI 63 63 + // preopens (which OpenFile(".", O_RDONLY)) don't issue an S3 call. 64 64 + key := strings.TrimPrefix(fs3.cleanPath(filename), "/") 65 65 + if key == "" || key == "." { 66 66 + return newS3DirFile(p, fs3.bucket, fs3.client), nil 67 67 + } 68 68 + 69 69 + f, err := newS3ReadFile(fs3.client, fs3.bucket, p) 70 70 + if err == nil { 71 71 + return f, nil 72 72 + } 73 73 + 74 74 + // If the object simply doesn't exist, the path may still be a 75 75 + // directory prefix in S3. Probe for that before giving up. 76 76 + var apiErr smithy.APIError 77 77 + if !errors.As(err, &apiErr) { 78 78 + return nil, err 79 79 + } 80 80 + switch apiErr.ErrorCode() { 81 81 + case "NoSuchKey", "NotFound": 82 82 + default: 83 83 + return nil, err 84 84 + } 85 85 + 86 86 + ctx := context.TODO() 87 87 + prefix := key + "/" 88 88 + maxKeys := int32(1) 89 89 + list, lerr := fs3.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ 90 90 + Bucket: &fs3.bucket, 91 91 + Prefix: &prefix, 92 92 + Delimiter: &fs3.separator, 93 93 + MaxKeys: &maxKeys, 94 94 + }) 95 95 + if lerr != nil { 96 96 + return nil, lerr 97 97 + } 98 98 + if len(list.Contents) > 0 || len(list.CommonPrefixes) > 0 { 99 99 + return newS3DirFile(p, fs3.bucket, fs3.client), nil 100 100 + } 101 101 + return nil, &os.PathError{Op: "open", Path: filename, Err: fs.ErrNotExist} 102 102 + 103 103 + case O_WRONLY: 104 104 + return newS3WriteFile(fs3.client, fs3.bucket, p) 105 105 + 106 106 + case O_WRMULTIPART: 107 107 + return newS3MultipartUploadFile(fs3.client, fs3.bucket, p) 108 108 + 109 109 + default: 110 110 + return nil, errors.New("unsupported open flag") 111 111 + } 112 112 + } 113 113 + 114 114 + // Stat returns a FileInfo describing the named file. 115 115 + func (fs3 *S3FS) Stat(filename string) (os.FileInfo, error) { 116 116 + key := strings.TrimPrefix(fs3.cleanPath(filename), "/") 117 117 + if key == "" || key == "." { 118 118 + return newDirInfo("/"), nil 119 119 + } 120 120 + 121 121 + ctx := context.TODO() 122 122 + 123 123 + head, err := fs3.client.HeadObject(ctx, &s3.HeadObjectInput{ 124 124 + Bucket: &fs3.bucket, 125 125 + Key: &key, 126 126 + }) 127 127 + if err == nil { 128 128 + return newFileInfo( 129 129 + path.Base(key), 130 130 + aws.ToInt64(head.ContentLength), 131 131 + aws.ToTime(head.LastModified), 132 132 + ), nil 133 133 + } 134 134 + 135 135 + var apiErr smithy.APIError 136 136 + if !errors.As(err, &apiErr) { 137 137 + return nil, err 138 138 + } 139 139 + switch apiErr.ErrorCode() { 140 140 + case "NotFound", "NoSuchKey": 141 141 + // fall through to directory probe below 142 142 + default: 143 143 + return nil, err 144 144 + } 145 145 + 146 146 + prefix := key + "/" 147 147 + maxKeys := int32(1) 148 148 + list, lerr := fs3.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ 149 149 + Bucket: &fs3.bucket, 150 150 + Prefix: &prefix, 151 151 + Delimiter: &fs3.separator, 152 152 + MaxKeys: &maxKeys, 153 153 + }) 154 154 + if lerr != nil { 155 155 + return nil, lerr 156 156 + } 157 157 + if len(list.Contents) > 0 || len(list.CommonPrefixes) > 0 { 158 158 + return newDirInfo(path.Base(key)), nil 159 159 + } 160 160 + return nil, &os.PathError{Op: "stat", Path: filename, Err: fs.ErrNotExist} 161 161 + } 162 162 + 163 163 + // Rename renames (moves) oldpath to newpath. If newpath already exists and 164 164 + // is not a directory, Rename replaces it. OS-specific restrictions may 165 165 + // apply when oldpath and newpath are in different directories. 166 166 + func (fs3 *S3FS) Rename(oldpath, newpath string) error { 167 167 + // TODO: Validate the paths? 168 168 + 169 169 + // Create a context 170 170 + ctx := context.TODO() // TODO: Get user-supplied context? 171 171 + 172 172 + // Format the paths 173 173 + src := path.Join(fs3.root, oldpath) 174 174 + dst := path.Join(fs3.root, newpath) 175 175 + 176 176 + // Send the copy request 177 177 + _, err := fs3.client.CopyObject(ctx, &s3.CopyObjectInput{ 178 178 + Bucket: &fs3.bucket, 179 179 + CopySource: &src, 180 180 + Key: &dst, 181 181 + }) 182 182 + if err != nil { 183 183 + return fmt.Errorf("failed to rename file: %s", err) 184 184 + } 185 185 + 186 186 + // Delete the old file 187 187 + // TODO: Parse the response? 188 188 + _, err = fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 189 189 + Bucket: &fs3.bucket, 190 190 + Key: &src, 191 191 + }) 192 192 + if err != nil { 193 193 + return fmt.Errorf("failed to remove file: %s", err) 194 194 + } 195 195 + 196 196 + return nil 197 197 + } 198 198 + 199 199 + // Remove removes the named file or directory. 200 200 + func (fs3 *S3FS) Remove(filename string) error { 201 201 + // TODO: Validate the path? 202 202 + // ... 203 203 + 204 204 + // Create a context 205 205 + ctx := context.TODO() // TODO: Get user-supplied context? 206 206 + 207 207 + // Format the path 208 208 + p := path.Join(fs3.root, filename) 209 209 + 210 210 + // Send the request 211 211 + // TODO: Parse the response? 212 212 + _, err := fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 213 213 + Bucket: &fs3.bucket, 214 214 + Key: &p, 215 215 + }) 216 216 + if err != nil { 217 217 + return fmt.Errorf("failed to remove file: %s", err) 218 218 + } 219 219 + return nil 220 220 + } 221 221 + 222 222 + // Join joins any number of path elements into a single path 223 223 + func (fs3 *S3FS) Join(elem ...string) string { 224 224 + j := path.Join(elem...) 225 225 + c := path.Clean(j) 226 226 + return c 227 227 + }

+29

internal/s3fs/chroot.go

Reviewed

··· 1 1 + // chroot.go implements the interface billy.Chroot 2 2 + 3 3 + package s3fs 4 4 + 5 5 + import "github.com/go-git/go-billy/v6" 6 6 + 7 7 + // Chroot returns a new filesystem from the same type where the new root is 8 8 + // the given path. Files outside of the designated directory tree cannot be 9 9 + // accessed. 10 10 + func (fs3 *S3FS) Chroot(path string) (billy.Filesystem, error) { 11 11 + // TODO: Check that path is a valid subdirectory of the current root 12 12 + // ... 13 13 + 14 14 + // Calculate the new root 15 15 + p := fs3.Join(fs3.root, path) 16 16 + 17 17 + // Create the new S3FS with the new root directory 18 18 + nfs := &S3FS{ 19 19 + client: fs3.client, 20 20 + bucket: fs3.bucket, 21 21 + root: p, 22 22 + } 23 23 + return nfs, nil 24 24 + } 25 25 + 26 26 + // Root returns the root path of the filesystem. 27 27 + func (fs3 *S3FS) Root() string { 28 28 + return fs3.root 29 29 + }

+84

internal/s3fs/dir.go

Reviewed

··· 1 1 + // dir.go implements the interface billy.Dir 2 2 + 3 3 + package s3fs 4 4 + 5 5 + import ( 6 6 + "context" 7 7 + "errors" 8 8 + "io/fs" 9 9 + "os" 10 10 + pathpkg "path" 11 11 + "strings" 12 12 + 13 13 + "github.com/aws/aws-sdk-go-v2/aws" 14 14 + "github.com/aws/aws-sdk-go-v2/service/s3" 15 15 + ) 16 16 + 17 17 + // ReadDir reads the directory named by dirname and returns a list of 18 18 + // directory entries sorted by filename. 19 19 + func (fs3 *S3FS) ReadDir(dir string) ([]fs.DirEntry, error) { 20 20 + key := strings.TrimPrefix(fs3.cleanPath(dir), "/") 21 21 + var prefix string 22 22 + if key != "" && key != "." { 23 23 + prefix = key + "/" 24 24 + } 25 25 + 26 26 + ctx := context.TODO() 27 27 + 28 28 + var ct *string 29 29 + var dirs []fs.DirEntry 30 30 + var files []fs.DirEntry 31 31 + for { 32 32 + res, err := fs3.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ 33 33 + Bucket: &fs3.bucket, 34 34 + Prefix: &prefix, 35 35 + ContinuationToken: ct, 36 36 + Delimiter: &fs3.separator, 37 37 + }) 38 38 + if err != nil { 39 39 + return nil, err 40 40 + } 41 41 + 42 42 + for _, d := range res.CommonPrefixes { 43 43 + name := strings.TrimSuffix(strings.TrimPrefix(aws.ToString(d.Prefix), prefix), "/") 44 44 + if name == "" { 45 45 + continue 46 46 + } 47 47 + dirs = append(dirs, fs.FileInfoToDirEntry(newDirInfo(name))) 48 48 + } 49 49 + 50 50 + for _, f := range res.Contents { 51 51 + full := aws.ToString(f.Key) 52 52 + if full == prefix { 53 53 + // zero-byte directory placeholder; skip 54 54 + continue 55 55 + } 56 56 + name := strings.TrimPrefix(full, prefix) 57 57 + if name == "" { 58 58 + continue 59 59 + } 60 60 + files = append(files, 61 61 + fs.FileInfoToDirEntry(newFileInfo( 62 62 + pathpkg.Base(name), 63 63 + aws.ToInt64(f.Size), 64 64 + aws.ToTime(f.LastModified), 65 65 + )), 66 66 + ) 67 67 + } 68 68 + 69 69 + if !aws.ToBool(res.IsTruncated) { 70 70 + break 71 71 + } 72 72 + ct = res.NextContinuationToken 73 73 + } 74 74 + 75 75 + return append(dirs, files...), nil 76 76 + } 77 77 + 78 78 + // MkdirAll creates a directory named path, along with any necessary 79 79 + // parents, and returns nil, or else returns an error. The permission bits 80 80 + // perm are used for all directories that MkdirAll creates. If path is/ 81 81 + // already a directory, MkdirAll does nothing and returns nil. 82 82 + func (fs3 *S3FS) MkdirAll(filename string, perm os.FileMode) error { 83 83 + return errors.New("not implemented") 84 84 + }

+452

internal/s3fs/file.go

Reviewed

··· 1 1 + package s3fs 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "context" 6 6 + "errors" 7 7 + "fmt" 8 8 + "io/fs" 9 9 + "io/ioutil" 10 10 + "os" 11 11 + "strings" 12 12 + "syscall" 13 13 + "time" 14 14 + 15 15 + "github.com/aws/aws-sdk-go-v2/service/s3" 16 16 + "github.com/tigrisdata/storage-go" 17 17 + "go.uber.org/atomic" 18 18 + ) 19 19 + 20 20 + const ( 21 21 + ModeMultipartUpload os.FileMode = fs.ModePerm + 1 // Custom os.FileMode for S3 multipart upload 22 22 + ) 23 23 + 24 24 + var ( 25 25 + ErrNotImplemented = errors.New("not implemented") 26 26 + ErrLockNotSupported = errors.New("lock not supported by s3") 27 27 + ErrTruncateNotSupported = errors.New("truncate not supported by s3") 28 28 + ErrFileClosed = errors.New("file is closed") 29 29 + ErrCantWriteToReadOnly = errors.New("can't write to read-only file") 30 30 + ErrCantReadFromWriteOnly = errors.New("can't read from write-only file") 31 31 + ) 32 32 + 33 33 + // s3ReadFile implements billy.File for S3, and represents a file opened in read mode. 34 34 + // 35 35 + // Upon creation, the file is loaded from S3. 36 36 + type s3ReadFile struct { 37 37 + client *storage.Client // S3 SDK client 38 38 + bucket string // S3 bucket name 39 39 + key string // File object's key in S3 40 40 + closed bool // Is the file closed? 41 41 + reader *bytes.Reader // Buffer for file contents 42 42 + head *s3.HeadObjectOutput // File metadata from S3 43 43 + } 44 44 + 45 45 + // newS3ReadFile creates a new s3ReadFile. 46 46 + func newS3ReadFile(client *storage.Client, bucket, key string) (*s3ReadFile, error) { 47 47 + // Create the context 48 48 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 49 49 + 50 50 + ho, err := client.HeadObject(ctx, &s3.HeadObjectInput{ 51 51 + Bucket: new(bucket), 52 52 + Key: new(key), 53 53 + }) 54 54 + if err != nil { 55 55 + return nil, &os.PathError{Op: "read", Path: key, Err: err} 56 56 + } 57 57 + 58 58 + // Run the GetObject operation 59 59 + res, err := client.GetObject(ctx, &s3.GetObjectInput{ 60 60 + Bucket: &bucket, 61 61 + Key: &key, 62 62 + }) 63 63 + if err != nil { 64 64 + return nil, fmt.Errorf("unable to perform GetObject operation: %w", err) 65 65 + } 66 66 + 67 67 + // Read the file contents and store in a bytes reader 68 68 + buf, err := ioutil.ReadAll(res.Body) 69 69 + if err != nil { 70 70 + return nil, fmt.Errorf("unable to read file body: %w", err) 71 71 + } 72 72 + reader := bytes.NewReader(buf) 73 73 + 74 74 + // Return the file 75 75 + return &s3ReadFile{ 76 76 + client: client, 77 77 + bucket: bucket, 78 78 + key: key, 79 79 + reader: reader, 80 80 + head: ho, 81 81 + }, nil 82 82 + } 83 83 + 84 84 + // Name returns the name of the file as presented to Open. 85 85 + func (f *s3ReadFile) Name() string { 86 86 + return f.key 87 87 + } 88 88 + 89 89 + // Write implements io.Writer for billy.File 90 90 + func (f *s3ReadFile) Write(p []byte) (n int, err error) { 91 91 + return 0, ErrCantWriteToReadOnly 92 92 + } 93 93 + 94 94 + // WriteAt implements io.WriterAt for billy.File 95 95 + func (f *s3ReadFile) WriteAt(p []byte, off int64) (n int, err error) { 96 96 + return 0, ErrCantWriteToReadOnly 97 97 + } 98 98 + 99 99 + // Read implements io.Reader for billy.File 100 100 + func (f *s3ReadFile) Read(p []byte) (n int, err error) { 101 101 + return f.reader.Read(p) 102 102 + } 103 103 + 104 104 + // ReadAt implements io.ReaderAt for billy.File 105 105 + func (f *s3ReadFile) ReadAt(p []byte, off int64) (n int, err error) { 106 106 + return f.reader.ReadAt(p, off) 107 107 + } 108 108 + 109 109 + // Seek implements io.Seeker for billy.File 110 110 + func (f *s3ReadFile) Seek(offset int64, whence int) (int64, error) { 111 111 + return f.reader.Seek(offset, whence) 112 112 + } 113 113 + 114 114 + // Close implements io.Closer for billy.File 115 115 + func (f *s3ReadFile) Close() error { 116 116 + // Was the file already closed? 117 117 + if f.closed { 118 118 + return ErrFileClosed 119 119 + } 120 120 + 121 121 + // Close the underlying file 122 122 + f.reader = nil 123 123 + 124 124 + // Mark the file as closed 125 125 + f.closed = true 126 126 + 127 127 + return nil 128 128 + } 129 129 + 130 130 + // Lock locks the file like e.g. flock. It protects against access from 131 131 + // other processes. 132 132 + func (f *s3ReadFile) Lock() error { 133 133 + return ErrLockNotSupported 134 134 + } 135 135 + 136 136 + // Unlock unlocks the file. 137 137 + func (f *s3ReadFile) Unlock() error { 138 138 + return ErrLockNotSupported 139 139 + } 140 140 + 141 141 + // Truncate the file. 142 142 + func (f *s3ReadFile) Truncate(size int64) error { 143 143 + return ErrTruncateNotSupported 144 144 + } 145 145 + 146 146 + func (f *s3ReadFile) Stat() (fs.FileInfo, error) { 147 147 + return enrichedFileInfo{ 148 148 + HeadObjectOutput: *f.head, 149 149 + key: f.key, 150 150 + mode: fs.ModePerm, 151 151 + }, nil 152 152 + } 153 153 + 154 154 + // s3WriteFile stores a file opened in write mode and implements billy.File 155 155 + // 156 156 + // Upon creation, a buffer is created to store the file contents. Upon close, 157 157 + // the file is uploaded to S3. 158 158 + type s3WriteFile struct { 159 159 + client *storage.Client // s3 skd client 160 160 + bucket string // S3 bucket name 161 161 + key string // File object's key in S3 162 162 + closed bool // Is the file closed? 163 163 + buf *bytes.Buffer // Buffer for storing the file before it's uploaded 164 164 + } 165 165 + 166 166 + // newS3WriteFile creates a new s3ReadFile. 167 167 + func newS3WriteFile(client *storage.Client, bucket, key string) (*s3WriteFile, error) { 168 168 + // TODO: Validate the key 169 169 + // ... 170 170 + 171 171 + return &s3WriteFile{ 172 172 + client: client, 173 173 + bucket: bucket, 174 174 + key: key, 175 175 + buf: bytes.NewBuffer(nil), 176 176 + }, nil 177 177 + } 178 178 + 179 179 + // Name returns the name of the file as presented to Open. 180 180 + func (f *s3WriteFile) Name() string { 181 181 + return f.key 182 182 + } 183 183 + 184 184 + // Write implements os.Writer for billy.File 185 185 + func (f *s3WriteFile) Write(p []byte) (n int, err error) { 186 186 + if f.closed { 187 187 + return 0, ErrFileClosed 188 188 + } 189 189 + return f.buf.Write(p) 190 190 + } 191 191 + 192 192 + func (f *s3WriteFile) WriteAt(p []byte, off int64) (n int, err error) { 193 193 + return 0, &os.PathError{Op: "write", Path: f.key, Err: ErrNotImplemented} 194 194 + } 195 195 + 196 196 + // Read implements os.Reader for billy.File 197 197 + func (f *s3WriteFile) Read(p []byte) (n int, err error) { 198 198 + return 0, ErrCantReadFromWriteOnly 199 199 + } 200 200 + 201 201 + // ReadAt implements io.ReaderAt for billy.File 202 202 + func (f *s3WriteFile) ReadAt(p []byte, off int64) (n int, err error) { 203 203 + return 0, ErrCantReadFromWriteOnly 204 204 + } 205 205 + 206 206 + // Seek implements io.Seeker for billy.File 207 207 + func (f *s3WriteFile) Seek(offset int64, whence int) (int64, error) { 208 208 + return 0, ErrNotImplemented 209 209 + } 210 210 + 211 211 + // Close implements io.Closer for billy.File 212 212 + func (f *s3WriteFile) Close() error { 213 213 + if f.closed { 214 214 + return ErrFileClosed 215 215 + } 216 216 + 217 217 + // Set to closed 218 218 + f.closed = true 219 219 + 220 220 + // Extract the body from the buffer 221 221 + body := bytes.NewReader(f.buf.Bytes()) 222 222 + 223 223 + // Create the context 224 224 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 225 225 + 226 226 + // Run the GetObject operation 227 227 + // TODO: Currently `res` is not used. Should it be? 228 228 + _, err := f.client.PutObject(ctx, &s3.PutObjectInput{ 229 229 + Bucket: &f.bucket, 230 230 + Key: &f.key, 231 231 + Body: body, 232 232 + }) 233 233 + if err != nil { 234 234 + return fmt.Errorf("unable to perform GetObject operation: %w", err) 235 235 + } 236 236 + 237 237 + return nil 238 238 + } 239 239 + 240 240 + func (f *s3WriteFile) Stat() (fs.FileInfo, error) { 241 241 + return newFileInfo(f.key, -1, time.Now()), nil 242 242 + } 243 243 + 244 244 + // Lock locks the file like e.g. flock. It protects against access from 245 245 + // other processes. 246 246 + func (f *s3WriteFile) Lock() error { 247 247 + return ErrLockNotSupported 248 248 + } 249 249 + 250 250 + // Unlock unlocks the file. 251 251 + func (f *s3WriteFile) Unlock() error { 252 252 + return ErrLockNotSupported 253 253 + } 254 254 + 255 255 + // Truncate the file. 256 256 + func (f *s3WriteFile) Truncate(size int64) error { 257 257 + return ErrTruncateNotSupported 258 258 + } 259 259 + 260 260 + // s3MultipartUploadFile implements billy.File 261 261 + type s3MultipartUploadFile struct { 262 262 + client *storage.Client // s3 skd client 263 263 + bucket string // S3 bucket name 264 264 + key string // File object's key in S3 265 265 + closed bool // Is the file closed? 266 266 + uploadID string // S3 multipart upload ID 267 267 + uploadN *atomic.Int32 // Counter tracking the number of uploads 268 268 + } 269 269 + 270 270 + // newS3MultipartUploadFile creates a new s3ReadFile. 271 271 + func newS3MultipartUploadFile(client *storage.Client, bucket, key string) (*s3MultipartUploadFile, error) { 272 272 + // TODO: Check if the file exists 273 273 + // ... 274 274 + 275 275 + // Create the context 276 276 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 277 277 + 278 278 + // Run the GetObject operation 279 279 + res, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ 280 280 + Bucket: &bucket, 281 281 + Key: &key, 282 282 + }) 283 283 + if err != nil { 284 284 + return nil, fmt.Errorf("unable to create multipart upload: %w", err) 285 285 + } 286 286 + 287 287 + // Return the file 288 288 + return &s3MultipartUploadFile{ 289 289 + client: client, 290 290 + bucket: bucket, 291 291 + key: key, 292 292 + uploadID: *res.UploadId, 293 293 + uploadN: atomic.NewInt32(1), 294 294 + }, nil 295 295 + } 296 296 + 297 297 + // Name returns the name of the file as presented to Open. 298 298 + func (f *s3MultipartUploadFile) Name() string { return f.key } 299 299 + 300 300 + // Write implements os.Writer for billy.File 301 301 + func (f *s3MultipartUploadFile) Write(p []byte) (n int, err error) { 302 302 + // Get the size of the data being written 303 303 + n = len(p) 304 304 + 305 305 + // Create a context for the operation 306 306 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 307 307 + 308 308 + // Create a reader for the data 309 309 + r := bytes.NewReader(p) 310 310 + 311 311 + // Get the part number 312 312 + pn := f.uploadN.Load() 313 313 + 314 314 + // Run the UploadPart operation 315 315 + _, err = f.client.UploadPart(ctx, &s3.UploadPartInput{ 316 316 + Bucket: &f.bucket, 317 317 + Key: &f.key, 318 318 + UploadId: &f.uploadID, 319 319 + PartNumber: new(pn), 320 320 + Body: r, 321 321 + }) 322 322 + if err != nil { 323 323 + return 0, fmt.Errorf("unable to upload part %d: %w", pn, err) 324 324 + } 325 325 + 326 326 + // Increment the part number 327 327 + f.uploadN.Add(1) 328 328 + 329 329 + // Return the number of bytes written 330 330 + return n, nil 331 331 + } 332 332 + 333 333 + func (f *s3MultipartUploadFile) WriteAt(p []byte, off int64) (n int, err error) { 334 334 + return 0, &os.PathError{Op: "write", Path: f.key, Err: ErrNotImplemented} 335 335 + } 336 336 + 337 337 + // Read implements os.Reader for billy.File 338 338 + func (f *s3MultipartUploadFile) Read(p []byte) (n int, err error) { 339 339 + return 0, ErrCantReadFromWriteOnly 340 340 + } 341 341 + 342 342 + // ReadAt implements io.ReaderAt for billy.File 343 343 + func (f *s3MultipartUploadFile) ReadAt(p []byte, off int64) (n int, err error) { 344 344 + return 0, ErrCantReadFromWriteOnly 345 345 + } 346 346 + 347 347 + // Seek implements io.Seeker for billy.File 348 348 + func (f *s3MultipartUploadFile) Seek(offset int64, whence int) (int64, error) { 349 349 + return 0, errors.New("seek not implemented") 350 350 + } 351 351 + 352 352 + // Close implements io.Closer for billy.File 353 353 + func (f *s3MultipartUploadFile) Close() error { 354 354 + // Check if the file has been closed 355 355 + if f.closed { 356 356 + return ErrFileClosed 357 357 + } 358 358 + 359 359 + // Set to closed 360 360 + f.closed = true 361 361 + 362 362 + // Create the context 363 363 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 364 364 + 365 365 + // Complete the multipart upload 366 366 + // TODO: Currently `res` is not used. Should it be? 367 367 + _, err := f.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ 368 368 + Bucket: &f.bucket, 369 369 + Key: &f.key, 370 370 + UploadId: &f.uploadID, 371 371 + }) 372 372 + if err != nil { 373 373 + return fmt.Errorf("unable to complete multipart upload: %w", err) 374 374 + } 375 375 + 376 376 + return nil 377 377 + } 378 378 + 379 379 + func (f *s3MultipartUploadFile) Lock() error { return ErrLockNotSupported } 380 380 + func (f *s3MultipartUploadFile) Unlock() error { return ErrLockNotSupported } 381 381 + func (f *s3MultipartUploadFile) Truncate(size int64) error { return ErrTruncateNotSupported } 382 382 + 383 383 + func (f *s3MultipartUploadFile) Stat() (fs.FileInfo, error) { 384 384 + return newFileInfo(f.key, -1, time.Now()), nil 385 385 + } 386 386 + 387 387 + // s3DirFile is a billy.File handle for a directory in S3. S3 has no real 388 388 + // directories, but WASI guests (via wazero) open the preopen root by calling 389 389 + // OpenFile(".", O_RDONLY) and then asking IsDir() — so we return a pseudo-file 390 390 + // that reports as a directory and rejects byte I/O with EISDIR. 391 391 + type s3DirFile struct { 392 392 + name, bucket string 393 393 + closed bool 394 394 + cli *storage.Client 395 395 + } 396 396 + 397 397 + func newS3DirFile(name, bucket string, cli *storage.Client) *s3DirFile { 398 398 + return &s3DirFile{ 399 399 + name: name, 400 400 + bucket: bucket, 401 401 + cli: cli, 402 402 + } 403 403 + } 404 404 + 405 405 + // Name returns the name of the file as presented to Open. 406 406 + func (f *s3DirFile) Name() string { 407 407 + return f.name 408 408 + } 409 409 + 410 410 + func (f *s3DirFile) eisdir(op string) error { 411 411 + return &os.PathError{Op: op, Path: f.name, Err: syscall.EISDIR} 412 412 + } 413 413 + 414 414 + func (f *s3DirFile) Read(p []byte) (int, error) { return 0, f.eisdir("read") } 415 415 + func (f *s3DirFile) ReadAt(p []byte, off int64) (int, error) { return 0, f.eisdir("read") } 416 416 + func (f *s3DirFile) Write(p []byte) (int, error) { return 0, f.eisdir("write") } 417 417 + func (f *s3DirFile) WriteAt(p []byte, off int64) (int, error) { return 0, f.eisdir("write") } 418 418 + func (f *s3DirFile) Seek(offset int64, whence int) (int64, error) { return 0, f.eisdir("seek") } 419 419 + func (f *s3DirFile) Truncate(size int64) error { return f.eisdir("truncate") } 420 420 + 421 421 + func (f *s3DirFile) Stat() (fs.FileInfo, error) { 422 422 + ho, err := f.cli.HeadObject(context.Background(), &s3.HeadObjectInput{ 423 423 + Bucket: new(f.bucket), 424 424 + Key: new(f.name), 425 425 + }) 426 426 + if err != nil { 427 427 + return nil, err 428 428 + } 429 429 + 430 430 + var mode fs.FileMode 431 431 + 432 432 + if strings.HasSuffix(f.name, "/") || *ho.ContentLength == 0 { 433 433 + mode = fs.ModeDir 434 434 + } 435 435 + 436 436 + return enrichedFileInfo{ 437 437 + HeadObjectOutput: *ho, 438 438 + key: f.name, 439 439 + mode: mode, 440 440 + }, nil 441 441 + } 442 442 + 443 443 + func (f *s3DirFile) Close() error { 444 444 + if f.closed { 445 445 + return ErrFileClosed 446 446 + } 447 447 + f.closed = true 448 448 + return nil 449 449 + } 450 450 + 451 451 + func (f *s3DirFile) Lock() error { return ErrLockNotSupported } 452 452 + func (f *s3DirFile) Unlock() error { return ErrLockNotSupported }

+79

internal/s3fs/file_test.go

Reviewed

··· 1 1 + package s3fs 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "errors" 6 6 + "testing" 7 7 + ) 8 8 + 9 9 + // TestS3WriteFileWrite locks in two invariants on the write path that a prior 10 10 + // stub silently broke: every Write must append its bytes to the buffer, and 11 11 + // writes after Close must return ErrFileClosed. The stub returned (0, nil) 12 12 + // for all writes regardless of state, which made every gzip/cp/etc. against 13 13 + // s3fs produce empty objects without surfacing an error. 14 14 + func TestS3WriteFileWrite(t *testing.T) { 15 15 + tests := []struct { 16 16 + name string 17 17 + closedPre bool 18 18 + chunks [][]byte 19 19 + wantBuf []byte 20 20 + wantErr error // expected error from the first Write (nil = all chunks succeed) 21 21 + }{ 22 22 + { 23 23 + name: "single chunk", 24 24 + chunks: [][]byte{[]byte("hello")}, 25 25 + wantBuf: []byte("hello"), 26 26 + }, 27 27 + { 28 28 + name: "multiple chunks accumulate", 29 29 + chunks: [][]byte{[]byte("hello "), []byte("world"), []byte("!")}, 30 30 + wantBuf: []byte("hello world!"), 31 31 + }, 32 32 + { 33 33 + name: "empty chunks are no-ops", 34 34 + chunks: [][]byte{[]byte("a"), {}, []byte("b")}, 35 35 + wantBuf: []byte("ab"), 36 36 + }, 37 37 + { 38 38 + name: "write after close returns ErrFileClosed", 39 39 + closedPre: true, 40 40 + chunks: [][]byte{[]byte("nope")}, 41 41 + wantBuf: nil, 42 42 + wantErr: ErrFileClosed, 43 43 + }, 44 44 + } 45 45 + 46 46 + for _, tt := range tests { 47 47 + t.Run(tt.name, func(t *testing.T) { 48 48 + f := &s3WriteFile{ 49 49 + bucket: "test", 50 50 + key: "k", 51 51 + buf: bytes.NewBuffer(nil), 52 52 + closed: tt.closedPre, 53 53 + } 54 54 + 55 55 + for i, c := range tt.chunks { 56 56 + n, err := f.Write(c) 57 57 + if i == 0 && tt.wantErr != nil { 58 58 + if !errors.Is(err, tt.wantErr) { 59 59 + t.Errorf("Write: err = %v, want %v", err, tt.wantErr) 60 60 + } 61 61 + if n != 0 { 62 62 + t.Errorf("Write: n = %d, want 0", n) 63 63 + } 64 64 + break 65 65 + } 66 66 + if err != nil { 67 67 + t.Fatalf("Write(%q): unexpected error: %v", c, err) 68 68 + } 69 69 + if n != len(c) { 70 70 + t.Fatalf("Write(%q): n=%d, want %d", c, n, len(c)) 71 71 + } 72 72 + } 73 73 + 74 74 + if got := f.buf.Bytes(); !bytes.Equal(got, tt.wantBuf) { 75 75 + t.Errorf("buffer = %q, want %q", got, tt.wantBuf) 76 76 + } 77 77 + }) 78 78 + } 79 79 + }

+54

internal/s3fs/fileinfo.go

Reviewed

··· 1 1 + package s3fs 2 2 + 3 3 + import ( 4 4 + "io/fs" 5 5 + "os" 6 6 + "time" 7 7 + 8 8 + "github.com/aws/aws-sdk-go-v2/service/s3" 9 9 + ) 10 10 + 11 11 + // simpleFileInfo implements os.FileInfo 12 12 + type simpleFileInfo struct { 13 13 + name string 14 14 + size int64 15 15 + mode os.FileMode 16 16 + modTime time.Time 17 17 + } 18 18 + 19 19 + func newFileInfo(name string, size int64, modTime time.Time) os.FileInfo { 20 20 + return simpleFileInfo{ 21 21 + name: name, 22 22 + size: size, 23 23 + mode: 0666, 24 24 + modTime: modTime, 25 25 + } 26 26 + } 27 27 + 28 28 + func newDirInfo(name string) os.FileInfo { 29 29 + return simpleFileInfo{ 30 30 + name: name, 31 31 + mode: fs.ModeDir, 32 32 + modTime: time.Now(), 33 33 + } 34 34 + } 35 35 + 36 36 + func (fi simpleFileInfo) Name() string { return fi.name } 37 37 + func (fi simpleFileInfo) Size() int64 { return fi.size } 38 38 + func (fi simpleFileInfo) Mode() os.FileMode { return fi.mode } 39 39 + func (fi simpleFileInfo) IsDir() bool { return fi.mode.IsDir() } 40 40 + func (fi simpleFileInfo) Sys() interface{} { return nil } 41 41 + func (fi simpleFileInfo) ModTime() time.Time { return fi.modTime } 42 42 + 43 43 + type enrichedFileInfo struct { 44 44 + s3.HeadObjectOutput 45 45 + key string 46 46 + mode fs.FileMode 47 47 + } 48 48 + 49 49 + func (tfi enrichedFileInfo) Name() string { return tfi.key } 50 50 + func (tfi enrichedFileInfo) Size() int64 { return *tfi.ContentLength } 51 51 + func (tfi enrichedFileInfo) Mode() fs.FileMode { return tfi.mode } 52 52 + func (tfi enrichedFileInfo) ModTime() time.Time { return *tfi.LastModified } 53 53 + func (tfi enrichedFileInfo) IsDir() bool { return tfi.mode.IsDir() } 54 54 + func (tfi enrichedFileInfo) Sys() any { return &tfi.HeadObjectOutput }

+53

internal/s3fs/filesystem.go

Reviewed

··· 1 1 + package s3fs 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + "path" 6 6 + 7 7 + "github.com/go-git/go-billy/v6" 8 8 + "github.com/tigrisdata/storage-go" 9 9 + ) 10 10 + 11 11 + const ( 12 12 + DefaultSeparator = "/" 13 13 + ) 14 14 + 15 15 + type S3FS struct { 16 16 + client *storage.Client 17 17 + bucket string 18 18 + root string 19 19 + separator string 20 20 + } 21 21 + 22 22 + // NewS3FS creates a new S3FS Filesystem. 23 23 + func NewS3FS(client *storage.Client, bucket string) (billy.Filesystem, error) { 24 24 + // Check for a non-nil client 25 25 + if client == nil { 26 26 + return nil, fmt.Errorf("s3 client cannot be nil") 27 27 + } 28 28 + return &S3FS{ 29 29 + client: client, 30 30 + bucket: bucket, 31 31 + root: "", 32 32 + separator: DefaultSeparator, 33 33 + }, nil 34 34 + } 35 35 + 36 36 + // Capabilities returns the filesystem capabilities. 37 37 + func (fs3 *S3FS) Capabilities() billy.Capability { 38 38 + return billy.ReadCapability | billy.WriteCapability 39 39 + } 40 40 + 41 41 + func (fs3 *S3FS) cleanPath(p ...string) string { 42 42 + // Join the path elements 43 43 + j := path.Join(p...) 44 44 + 45 45 + // Clean the path before joining to root 46 46 + c := path.Clean(j) 47 47 + 48 48 + // Join the root and cleaned path 49 49 + f := path.Join(fs3.root, c) 50 50 + 51 51 + // Return the full path 52 52 + return path.Clean(f) 53 53 + }

+37

internal/s3fs/symlink.go

Reviewed

··· 1 1 + // symlink.go implements the interface billy.Symlink 2 2 + 3 3 + package s3fs 4 4 + 5 5 + import ( 6 6 + "errors" 7 7 + "os" 8 8 + ) 9 9 + 10 10 + var ( 11 11 + ErrSymLinkNotSupported = errors.New("symlink not supported by s3") 12 12 + ) 13 13 + 14 14 + // Lstat returns a FileInfo describing the named file. S3 has no symlinks, 15 15 + // so Lstat is equivalent to Stat. We still implement it so callers that 16 16 + // type-assert billy.Symlink (rm, du, cp, ls, touch, file, billyfs, 17 17 + // billysh) get a usable FileInfo instead of ErrSymLinkNotSupported. 18 18 + func (fs3 *S3FS) Lstat(filename string) (os.FileInfo, error) { 19 19 + return fs3.Stat(filename) 20 20 + } 21 21 + 22 22 + // Symlink creates a symbolic-link from link to target. target may be an 23 23 + // absolute or relative path, and need not refer to an existing node. 24 24 + // Parent directories of link are created as necessary. 25 25 + // 26 26 + // NOTE: Symlink is not supported by s3. It always returns an error. 27 27 + func (fs3 *S3FS) Symlink(target, link string) error { 28 28 + return ErrSymLinkNotSupported 29 29 + } 30 30 + 31 31 + // Readlink returns the target path of link. 32 32 + // 33 33 + // NOTE: Readlink is not supported by s3. It always returns an error. 34 34 + // (This may be revised in the future.) 35 35 + func (fs3 *S3FS) Readlink(link string) (string, error) { 36 36 + return "", ErrSymLinkNotSupported 37 37 + }

+17

internal/s3fs/tempfile.go

Reviewed

··· 1 1 + // tempfile.go implements the interface billy.TempFile 2 2 + 3 3 + package s3fs 4 4 + 5 5 + import "github.com/go-git/go-billy/v6" 6 6 + 7 7 + // TempFile creates a new temporary file in the directory dir with a name 8 8 + // beginning with prefix, opens the file for reading and writing, and 9 9 + // returns the resulting *os.File. If dir is the empty string, TempFile 10 10 + // uses the default directory for temporary files (see os.TempDir). 11 11 + // Multiple programs calling TempFile simultaneously will not choose the 12 12 + // same file. The caller can use f.Name() to find the pathname of the file. 13 13 + // It is the caller's responsibility to remove the file when no longer 14 14 + // needed. 15 15 + func (fs3 *S3FS) TempFile(dir, prefix string) (billy.File, error) { 16 16 + return nil, nil 17 17 + }