From d2281c8b0cb4fb84b4a8f4ae4c5357221f1a8bea Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 5 Mar 2024 17:21:06 +0000 Subject: [PATCH] s3: support metadata setting and mapping on server side Copy Before this change the backend would not run the metadata mapper and it would ignore metadata set when doing server side copies. --- backend/s3/gen_setfrom.go | 1 + backend/s3/s3.go | 22 +++++++++++++++++---- backend/s3/setfrom.go | 40 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/backend/s3/gen_setfrom.go b/backend/s3/gen_setfrom.go index b3e0247f2..b70615283 100644 --- a/backend/s3/gen_setfrom.go +++ b/backend/s3/gen_setfrom.go @@ -98,4 +98,5 @@ import "github.com/aws/aws-sdk-go/service/s3" genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput)) genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput)) genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput)) + genSetFrom(new(s3.CopyObjectInput), new(s3.PutObjectInput)) } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 9183670c4..63f1f46ba 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4591,10 +4591,22 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, fs.Debugf(src, "Can't copy - not same remote type") return nil, fs.ErrorCantCopy } + srcBucket, srcPath := srcObj.split() req := s3.CopyObjectInput{ MetadataDirective: aws.String(s3.MetadataDirectiveCopy), } + + // Update the metadata if it is in use + if ci := fs.GetConfig(ctx); ci.Metadata { + ui, err := srcObj.prepareUpload(ctx, src, fs.MetadataAsOpenOptions(ctx), true) + if err != nil { + return nil, fmt.Errorf("failed to prepare upload: %w", err) + } + setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req) + req.MetadataDirective = aws.String(s3.MetadataDirectiveReplace) + } + err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj) if err != nil { return nil, err @@ -5697,7 +5709,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn fs: f, remote: remote, } - ui, err := o.prepareUpload(ctx, src, options) + ui, err := o.prepareUpload(ctx, src, options, false) if err != nil { return info, nil, fmt.Errorf("failed to prepare upload: %w", err) } @@ -6064,7 +6076,9 @@ type uploadInfo struct { } // Prepare object for being uploaded -func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (ui uploadInfo, err error) { +// +// If noHash is true the md5sum will not be calculated +func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, noHash bool) (ui uploadInfo, err error) { bucket, bucketPath := o.split() // Create parent dir/bucket if not saving directory marker if !strings.HasSuffix(o.remote, "/") { @@ -6138,7 +6152,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [ var md5sumBase64 string size := src.Size() multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff) - if !multipart || !o.fs.opt.DisableChecksum { + if !noHash && (!multipart || !o.fs.opt.DisableChecksum) { ui.md5sumHex, err = src.Hash(ctx, hash.MD5) if err == nil && matchMd5.MatchString(ui.md5sumHex) { hashBytes, err := hex.DecodeString(ui.md5sumHex) @@ -6250,7 +6264,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if multipart { wantETag, gotETag, versionID, ui, err = o.uploadMultipart(ctx, src, in, options...) } else { - ui, err = o.prepareUpload(ctx, src, options) + ui, err = o.prepareUpload(ctx, src, options, false) if err != nil { return fmt.Errorf("failed to prepare upload: %w", err) } diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.go index 1f7000b24..27eb2aafe 100644 --- a/backend/s3/setfrom.go +++ b/backend/s3/setfrom.go @@ -11,6 +11,7 @@ func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b * a.EncodingType = b.EncodingType a.ExpectedBucketOwner = b.ExpectedBucketOwner a.MaxKeys = b.MaxKeys + a.OptionalObjectAttributes = b.OptionalObjectAttributes a.Prefix = b.Prefix a.RequestPayer = b.RequestPayer } @@ -25,6 +26,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output a.MaxKeys = b.MaxKeys a.Name = b.Name a.Prefix = b.Prefix + a.RequestCharged = b.RequestCharged } // setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b @@ -34,7 +36,9 @@ func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVers a.EncodingType = b.EncodingType a.ExpectedBucketOwner = b.ExpectedBucketOwner a.MaxKeys = b.MaxKeys + a.OptionalObjectAttributes = b.OptionalObjectAttributes a.Prefix = b.Prefix + a.RequestPayer = b.RequestPayer } // setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b @@ -55,6 +59,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV a.MaxKeys = b.MaxKeys a.Name = b.Name a.Prefix = b.Prefix + a.RequestCharged = b.RequestCharged } // setFrom_s3Object_s3ObjectVersion copies matching elements from a to b @@ -64,6 +69,7 @@ func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) { a.Key = b.Key a.LastModified = b.LastModified a.Owner = b.Owner + a.RestoreStatus = b.RestoreStatus a.Size = b.Size a.StorageClass = b.StorageClass } @@ -237,3 +243,37 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P a.StorageClass = b.StorageClass a.WebsiteRedirectLocation = b.WebsiteRedirectLocation } + +// setFrom_s3CopyObjectInput_s3PutObjectInput copies matching elements from a to b +func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.PutObjectInput) { + a.ACL = b.ACL + a.Bucket = b.Bucket + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.Expires = b.Expires + a.GrantFullControl = b.GrantFullControl + a.GrantRead = b.GrantRead + a.GrantReadACP = b.GrantReadACP + a.GrantWriteACP = b.GrantWriteACP + a.Key = b.Key + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.Tagging = b.Tagging + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +}