From f3e982d3bfea664d3cc8ef92be8f16698aebaa95 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 1 Mar 2018 12:11:34 +0000 Subject: [PATCH] azureblob,b2,gcs,qingstor,s3,swift: Don't check for bucket/container presense if listing was OK In a typical rclone copy to a bucket/container based remote, before this change we were doing a list, followed by a HEAD of the bucket to check it existed before doing the copy. The fact the list succeeded means the bucket exists so mark it OK at that point. Issue #1421 --- backend/azureblob/azureblob.go | 14 ++++++++++++++ backend/b2/b2.go | 13 +++++++++++++ backend/googlecloudstorage/googlecloudstorage.go | 13 +++++++++++++ backend/qingstor/qingstor.go | 14 ++++++++++++++ backend/s3/s3.go | 14 ++++++++++++++ backend/swift/swift.go | 13 +++++++++++++ 6 files changed, 81 insertions(+) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 2157759bb..6b4f6b8e8 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -405,6 +405,16 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Blob, isDirectory boo return o, nil } +// mark the container as being OK +func (f *Fs) markContainerOK() { + if f.container != "" { + f.containerOKMu.Lock() + f.containerOK = true + f.containerDeleted = false + f.containerOKMu.Unlock() + } +} + // listDir lists a single directory func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { err = f.list(dir, false, listChunkSize, func(remote string, object *storage.Blob, isDirectory bool) error { @@ -420,6 +430,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // container must be present if listing succeeded + f.markContainerOK() return entries, nil } @@ -491,6 +503,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // container must be present if listing succeeded + f.markContainerOK() return list.Flush() } diff --git a/backend/b2/b2.go b/backend/b2/b2.go index f68bcbfe6..cf2ac601d 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -551,6 +551,15 @@ func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, l return o, nil } +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketOKMu.Unlock() + } +} + // listDir lists a single directory func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { last := "" @@ -567,6 +576,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // bucket must be present if listing succeeded + f.markBucketOK() return entries, nil } @@ -634,6 +645,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // bucket must be present if listing succeeded + f.markBucketOK() return list.Flush() } diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 07c58fe1f..e16518835 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -463,6 +463,15 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory b return o, nil } +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketOKMu.Unlock() + } +} + // listDir lists a single directory func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { // List the objects @@ -479,6 +488,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // bucket must be present if listing succeeded + f.markBucketOK() return entries, err } @@ -555,6 +566,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // bucket must be present if listing succeeded + f.markBucketOK() return list.Flush() } diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index 2b5213410..fa02b4ea0 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -519,6 +519,16 @@ func (f *Fs) itemToDirEntry(remote string, object *qs.KeyType, isDirectory bool) return o, nil } +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketDeleted = false + f.bucketOKMu.Unlock() + } +} + // listDir lists files and directories to out func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { // List the objects and directories @@ -535,6 +545,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // bucket must be present if listing succeeded + f.markBucketOK() return entries, nil } @@ -606,6 +618,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // bucket must be present if listing succeeded + f.markBucketOK() return list.Flush() } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 0aa409172..311a38863 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -590,6 +590,16 @@ func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool) return o, nil } +// mark the bucket as being OK +func (f *Fs) markBucketOK() { + if f.bucket != "" { + f.bucketOKMu.Lock() + f.bucketOK = true + f.bucketDeleted = false + f.bucketOKMu.Unlock() + } +} + // listDir lists files and directories to out func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { // List the objects and directories @@ -606,6 +616,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // bucket must be present if listing succeeded + f.markBucketOK() return entries, nil } @@ -673,6 +685,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // bucket must be present if listing succeeded + f.markBucketOK() return list.Flush() } diff --git a/backend/swift/swift.go b/backend/swift/swift.go index c45ef4ee2..c33d4c5e5 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -407,6 +407,15 @@ func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error { return err } +// mark the container as being OK +func (f *Fs) markContainerOK() { + if f.container != "" { + f.containerOKMu.Lock() + f.containerOK = true + f.containerOKMu.Unlock() + } +} + // listDir lists a single directory func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if f.container == "" { @@ -420,6 +429,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { if err != nil { return nil, err } + // container must be present if listing succeeded + f.markContainerOK() return entries, nil } @@ -482,6 +493,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { if err != nil { return err } + // container must be present if listing succeeded + f.markContainerOK() return list.Flush() }