github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-26 04:07:22 +01:00
Code Releases Activity

s3: fix download of compressed files from Cloudflare R2 - fixes #8137

Browse Source 
Before this change attempting to download a file with
`Content-Encoding: gzip` from Cloudflare R2 gave this error

    corrupted on transfer: sizes differ src 0 vs dst 999

This was caused by the SDK v2 overriding our attempt to set
`Accept-Encoding: gzip`.

This fixes the problem by disabling the middleware that does that
overriding.
This commit is contained in:
Nick Craig-Wood 2024-10-22 20:12:18 +01:00
parent bf57087a6e
commit 6079cab090
2 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backend/s3/s3.go
View File

@ -5866,6 +5866,25 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
return resp.Body, err
}
// middleware to stop the SDK adding `Accept-Encoding: identity`
func removeDisableGzip() func(*middleware.Stack) error {
return func(stack *middleware.Stack) error {
_, err := stack.Finalize.Remove("DisableAcceptEncodingGzip")
return err
}
}
// middleware to set Accept-Encoding to how we want it
//
// This make sure we download compressed files as-is from all platforms
func (f *Fs) acceptEncoding() (APIOptions []func(*middleware.Stack) error) {
APIOptions = append(APIOptions, removeDisableGzip())
if f.opt.UseAcceptEncodingGzip.Value {
APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
}
return APIOptions
}
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
bucket, bucketPath := o.split()
@ -5899,11 +5918,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var APIOptions []func(*middleware.Stack) error
// Override the automatic decompression in the transport to
// download compressed files as-is
if o.fs.opt.UseAcceptEncodingGzip.Value {
APIOptions = append(APIOptions, smithyhttp.AddHeaderValue("Accept-Encoding", "gzip"))
}
// Set the SDK to always download compressed files as-is
APIOptions = append(APIOptions, o.fs.acceptEncoding()...)
for _, option := range options {
switch option.(type) {

3
docs/content/s3.md
View File

@ -2503,6 +2503,9 @@ Note that Cloudflare decompresses files uploaded with
does. If this is causing a problem then upload the files with
`--header-upload "Cache-Control: no-transform"`
A consequence of this is that `Content-Encoding: gzip` will never
appear in the metadata on Cloudflare.
### Dreamhost
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is