diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 63c2cac6d..0e2e0d8ee 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2196,6 +2196,15 @@ See: https://github.com/rclone/rclone/issues/4673, https://github.com/rclone/rcl This is usually set to a CloudFront CDN URL as AWS S3 offers cheaper egress for data downloaded through the CloudFront network.`, Advanced: true, + }, { + Name: "directory_markers", + Default: false, + Advanced: true, + Help: `Upload an empty object with a trailing slash in name when new directory is created + +Empty folders are unsupported for bucket based remotes, this option creates an empty +object named "/", to persist folder. +`, }, { Name: "use_multipart_etag", Help: `Whether to use ETag in multipart uploads for verification @@ -2425,6 +2434,7 @@ type Options struct { MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"` DisableHTTP2 bool `config:"disable_http2"` DownloadURL string `config:"download_url"` + DirectoryMarkers bool `config:"directory_markers"` UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` UsePresignedRequest bool `config:"use_presigned_request"` Versions bool `config:"versions"` @@ -3879,7 +3889,27 @@ func (f *Fs) bucketExists(ctx context.Context, bucket string) (bool, error) { // Mkdir creates the bucket if it doesn't exist func (f *Fs) Mkdir(ctx context.Context, dir string) error { bucket, _ := f.split(dir) - return f.makeBucket(ctx, bucket) + e := f.makeBucket(ctx, bucket) + if e != nil { + return e + } + // Create directory marker file + if f.opt.DirectoryMarkers && bucket != "" && dir != "" { + markerFilePath := fmt.Sprintf("%s/", dir) + markerFileContent := io.Reader(strings.NewReader("")) + markerFileObject := &Object{ + fs: f, + remote: markerFilePath, + meta: map[string]string{ + metaMtime: swift.TimeToFloatString(time.Now()), + }, + } + _, e := f.Put(ctx, markerFileContent, markerFileObject) + if e != nil { + return e + } + } + return nil } // makeBucket creates the bucket if it doesn't exist @@ -3920,6 +3950,15 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error { // Returns an error if it isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { bucket, directory := f.split(dir) + // Remove directory marker file + if f.opt.DirectoryMarkers && bucket != "" && dir != "" { + markerFilePath := fmt.Sprintf("%s/", dir) + markerFileObject := &Object{ + fs: f, + remote: markerFilePath, + } + _ = markerFileObject.Remove(ctx) + } if bucket == "" || directory != "" { return nil } diff --git a/backend/s3/s3_test.go b/backend/s3/s3_test.go index e226878a5..2cec88185 100644 --- a/backend/s3/s3_test.go +++ b/backend/s3/s3_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" ) @@ -20,6 +21,24 @@ func TestIntegration(t *testing.T) { }) } +func TestIntegration2(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("skipping as -remote is set") + } + name := "TestS3" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":", + NilObject: (*Object)(nil), + TiersToTest: []string{"STANDARD", "STANDARD_IA"}, + ChunkedUpload: fstests.ChunkedUploadConfig{ + MinChunkSize: minChunkSize, + }, + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "directory_markers", Value: "true"}, + }, + }) +} + func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) { return f.setUploadChunkSize(cs) }