From aaa1370a3646db41265a76de89a58917f75d4576 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 25 Nov 2016 21:52:43 +0000 Subject: [PATCH] Add directory parameter to Rmdir and Mkdir #100 #831 This will enable rclone to manage directories properly in the future. --- amazonclouddrive/amazonclouddrive.go | 28 +++++++++++------ amazonclouddrive/amazonclouddrive_test.go | 1 + b2/b2.go | 12 ++++--- b2/b2_test.go | 1 + cmd/mkdir/mkdir.go | 2 +- cmd/mount/fs_test.go | 2 +- cmd/rmdir/rmdir.go | 2 +- crypt/crypt2_test.go | 1 + crypt/crypt_test.go | 1 + dircache/dircache.go | 29 +++++++++++++++++ drive/drive.go | 31 +++++++++++++------ drive/drive_test.go | 1 + dropbox/dropbox.go | 15 +++++---- dropbox/dropbox_test.go | 1 + fs/fs.go | 4 +-- fs/operations.go | 14 ++++----- fs/operations_test.go | 2 +- fs/sync.go | 2 +- fs/sync_test.go | 2 +- fstest/fstest.go | 25 +++++++++++---- fstest/fstests/fstests.go | 21 +++++++++++-- googlecloudstorage/googlecloudstorage.go | 10 ++++-- googlecloudstorage/googlecloudstorage_test.go | 1 + hubic/hubic_test.go | 1 + local/local.go | 19 +++++++----- local/local_test.go | 1 + onedrive/onedrive.go | 30 +++++++++++------- onedrive/onedrive_test.go | 1 + s3/s3.go | 10 ++++-- s3/s3_test.go | 1 + swift/swift.go | 12 ++++--- swift/swift_test.go | 1 + yandex/yandex.go | 24 +++++++++----- yandex/yandex_test.go | 1 + 34 files changed, 220 insertions(+), 89 deletions(-) diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index 0ddd385d3..95cab7746 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -18,6 +18,7 @@ import ( "io" "log" "net/http" + "path" "regexp" "strings" "sync/atomic" @@ -607,8 +608,15 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { - return f.dirCache.FindRoot(true) +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err } // Move src to this remote using server side move operations. @@ -685,16 +693,16 @@ func (f *Fs) DirMove(src fs.Fs) (err error) { // purgeCheck remotes the root directory, if check is set then it // refuses to do so if it has anything in -func (f *Fs) purgeCheck(check bool) error { - if f.root == "" { +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { return errors.New("can't purge root directory") } dc := f.dirCache - err := dc.FindRoot(false) + rootID, err := dc.FindDir(dir, false) if err != nil { return err } - rootID := dc.RootID() if check { // check directory is empty @@ -730,7 +738,7 @@ func (f *Fs) purgeCheck(check bool) error { return err } - f.dirCache.ResetRoot() + f.dirCache.FlushDir(dir) if err != nil { return err } @@ -740,8 +748,8 @@ func (f *Fs) purgeCheck(check bool) error { // Rmdir deletes the root folder // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - return f.purgeCheck(true) +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) } // Precision return the precision of this Fs @@ -783,7 +791,7 @@ func (f *Fs) Hashes() fs.HashSet { // deleting all the files quicker than just running Remove() on the // result of List() func (f *Fs) Purge() error { - return f.purgeCheck(false) + return f.purgeCheck("", false) } // ------------------------------------------------------------ diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go index 0fc3d99f8..657afc1dd 100644 --- a/amazonclouddrive/amazonclouddrive_test.go +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/b2/b2.go b/b2/b2.go index cffc29950..149ab777b 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -745,7 +745,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the bucket if it doesn't exist -func (f *Fs) Mkdir() error { +func (f *Fs) Mkdir(dir string) error { + // Can't create subdirs + if dir != "" { + return nil + } opts := rest.Opts{ Method: "POST", Path: "/b2_create_bucket", @@ -784,8 +788,8 @@ func (f *Fs) Mkdir() error { // Rmdir deletes the bucket if the fs is at the root // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - if f.root != "" { +func (f *Fs) Rmdir(dir string) error { + if f.root != "" || dir != "" { return nil } opts := rest.Opts{ @@ -896,7 +900,7 @@ func (f *Fs) purge(oldOnly bool) error { wg.Wait() if !oldOnly { - checkErr(f.Rmdir()) + checkErr(f.Rmdir("")) } return errReturn } diff --git a/b2/b2_test.go b/b2/b2_test.go index c06f633df..d3bc2546a 100644 --- a/b2/b2_test.go +++ b/b2/b2_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/cmd/mkdir/mkdir.go b/cmd/mkdir/mkdir.go index bf6a1e1ea..73ec83513 100644 --- a/cmd/mkdir/mkdir.go +++ b/cmd/mkdir/mkdir.go @@ -17,7 +17,7 @@ var mkdirCmd = &cobra.Command{ cmd.CheckArgs(1, 1, command, args) fdst := cmd.NewFsDst(args) cmd.Run(true, command, func() error { - return fs.Mkdir(fdst) + return fs.Mkdir(fdst, "") }) }, } diff --git a/cmd/mount/fs_test.go b/cmd/mount/fs_test.go index 7d202fa0c..4c8b3bdea 100644 --- a/cmd/mount/fs_test.go +++ b/cmd/mount/fs_test.go @@ -82,7 +82,7 @@ func newRun() *Run { log.Fatalf("Failed to open remote %q: %v", *RemoteName, err) } - err = r.fremote.Mkdir() + err = r.fremote.Mkdir("") if err != nil { log.Fatalf("Failed to open mkdir %q: %v", *RemoteName, err) } diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go index 0aeb361c6..1235d53d0 100644 --- a/cmd/rmdir/rmdir.go +++ b/cmd/rmdir/rmdir.go @@ -20,7 +20,7 @@ objects in it, use purge for that.`, cmd.CheckArgs(1, 1, command, args) fdst := cmd.NewFsDst(args) cmd.Run(true, command, func() error { - return fs.Rmdir(fdst) + return fs.Rmdir(fdst, "") }) }, } diff --git a/crypt/crypt2_test.go b/crypt/crypt2_test.go index 8a19292c0..0af830b05 100644 --- a/crypt/crypt2_test.go +++ b/crypt/crypt2_test.go @@ -24,6 +24,7 @@ func TestFsString2(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty2(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound2(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir2(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir2(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty2(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty2(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound2(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go index fbbef92fb..7a96b7bb2 100644 --- a/crypt/crypt_test.go +++ b/crypt/crypt_test.go @@ -24,6 +24,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/dircache/dircache.go b/dircache/dircache.go index 2a221a7b6..7bd971267 100644 --- a/dircache/dircache.go +++ b/dircache/dircache.go @@ -78,6 +78,35 @@ func (dc *DirCache) Flush() { dc.cacheMu.Unlock() } +// FlushDir flushes the map of all data starting with dir +// +// If dir is empty then this is equivalent to calling ResetRoot +func (dc *DirCache) FlushDir(dir string) { + if dir == "" { + dc.ResetRoot() + return + } + dc.cacheMu.Lock() + + // Delete the root dir + ID, ok := dc.cache[dir] + if ok { + delete(dc.cache, dir) + delete(dc.invCache, ID) + } + + // And any sub directories + dir += "/" + for key, ID := range dc.cache { + if strings.HasPrefix(key, dir) { + delete(dc.cache, key) + delete(dc.invCache, ID) + } + } + + dc.cacheMu.Unlock() +} + // SplitPath splits a path into directory, leaf // // Path shouldn't start or end with a / diff --git a/drive/drive.go b/drive/drive.go index 98d3edd11..c362ee6ae 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -12,6 +12,7 @@ import ( "io" "log" "net/http" + "path" "strings" "time" @@ -592,21 +593,30 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { - return f.dirCache.FindRoot(true) +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err } // Rmdir deletes the container // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - err := f.dirCache.FindRoot(false) +func (f *Fs) Rmdir(dir string) error { + root := path.Join(f.root, dir) + dc := f.dirCache + rootID, err := dc.FindDir(dir, false) if err != nil { return err } var children *drive.ChildList err = f.pacer.Call(func() (bool, error) { - children, err = f.svc.Children.List(f.dirCache.RootID()).MaxResults(10).Do() + children, err = f.svc.Children.List(rootID).MaxResults(10).Do() return shouldRetry(err) }) if err != nil { @@ -616,12 +626,12 @@ func (f *Fs) Rmdir() error { return errors.Errorf("directory not empty: %#v", children.Items) } // Delete the directory if it isn't the root - if f.root != "" { + if root != "" { err = f.pacer.Call(func() (bool, error) { if *driveUseTrash { - _, err = f.svc.Files.Trash(f.dirCache.RootID()).Do() + _, err = f.svc.Files.Trash(rootID).Do() } else { - err = f.svc.Files.Delete(f.dirCache.RootID()).Do() + err = f.svc.Files.Delete(rootID).Do() } return shouldRetry(err) }) @@ -629,7 +639,10 @@ func (f *Fs) Rmdir() error { return err } } - f.dirCache.ResetRoot() + f.dirCache.FlushDir(dir) + if err != nil { + return err + } return nil } diff --git a/drive/drive_test.go b/drive/drive_test.go index 0d3db28d0..1b0f50d22 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 9c0e86f36..b7389dd4c 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -448,30 +448,33 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { - entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit) +func (f *Fs) Mkdir(dir string) error { + root := path.Join(f.slashRoot, dir) + entry, err := f.db.Metadata(root, false, false, "", "", metadataLimit) if err == nil { if entry.IsDir { return nil } return errors.Errorf("%q already exists as file", f.root) } - _, err = f.db.CreateFolder(f.slashRoot) + _, err = f.db.CreateFolder(root) return err } // Rmdir deletes the container // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - entry, err := f.db.Metadata(f.slashRoot, true, false, "", "", 16) +func (f *Fs) Rmdir(dir string) error { + root := path.Join(f.slashRoot, dir) + entry, err := f.db.Metadata(root, true, false, "", "", 16) if err != nil { return err } if len(entry.Contents) != 0 { return errors.New("directory not empty") } - return f.Purge() + _, err = f.db.Delete(root) + return err } // Precision returns the precision diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 8f1c31391..3d5b87a06 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/fs/fs.go b/fs/fs.go index ea7d00340..592245595 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -135,12 +135,12 @@ type Fs interface { // Mkdir makes the directory (container, bucket) // // Shouldn't return an error if it already exists - Mkdir() error + Mkdir(dir string) error // Rmdir removes the directory (container, bucket) if empty // // Return an error if it doesn't exist or isn't empty - Rmdir() error + Rmdir(dir string) error } // Info provides an interface to reading information about a filesystem. diff --git a/fs/operations.go b/fs/operations.go index 4647161c1..230d089f0 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -707,12 +707,12 @@ func ListDir(f Fs, w io.Writer) error { } // Mkdir makes a destination directory or container -func Mkdir(f Fs) error { +func Mkdir(f Fs, dir string) error { if Config.DryRun { Log(f, "Not making directory as dry run is set") return nil } - err := f.Mkdir() + err := f.Mkdir(dir) if err != nil { Stats.Error() return err @@ -722,17 +722,17 @@ func Mkdir(f Fs) error { // TryRmdir removes a container but not if not empty. It doesn't // count errors but may return one. -func TryRmdir(f Fs) error { +func TryRmdir(f Fs, dir string) error { if Config.DryRun { Log(f, "Not deleting as dry run is set") return nil } - return f.Rmdir() + return f.Rmdir(dir) } // Rmdir removes a container but not if not empty -func Rmdir(f Fs) error { - err := TryRmdir(f) +func Rmdir(f Fs, dir string) error { + err := TryRmdir(f, dir) if err != nil { Stats.Error() return err @@ -764,7 +764,7 @@ func Purge(f Fs) error { if err != nil { return err } - err = Rmdir(f) + err = Rmdir(f, "") } if err != nil { Stats.Error() diff --git a/fs/operations_test.go b/fs/operations_test.go index 9ed727e06..06b793856 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -201,7 +201,7 @@ func (r *Run) WriteObjectTo(f fs.Fs, remote, content string, modTime time.Time, } const maxTries = 10 if !r.mkdir[f.String()] { - err := f.Mkdir() + err := f.Mkdir("") if err != nil { r.Fatalf("Failed to mkdir %q: %v", f, err) } diff --git a/fs/sync.go b/fs/sync.go index 8925414b2..11cac2d7c 100644 --- a/fs/sync.go +++ b/fs/sync.go @@ -402,7 +402,7 @@ func (s *syncCopyMove) run() error { return nil } - err := Mkdir(s.fdst) + err := Mkdir(s.fdst, "") if err != nil { return err } diff --git a/fs/sync_test.go b/fs/sync_test.go index d06c1703b..84fbfc70c 100644 --- a/fs/sync_test.go +++ b/fs/sync_test.go @@ -120,7 +120,7 @@ func TestCopyAfterDelete(t *testing.T) { fstest.CheckItems(t, r.flocal) fstest.CheckItems(t, r.fremote, file1) - err := fs.Mkdir(r.flocal) + err := fs.Mkdir(r.flocal, "") require.NoError(t, err) err = fs.CopyDir(r.fremote, r.flocal) diff --git a/fstest/fstest.go b/fstest/fstest.go index 61af1f628..de68dbf61 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -14,6 +14,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "testing" "time" @@ -144,15 +145,16 @@ func (is *Items) Done(t *testing.T) { // CheckListingWithPrecision checks the fs to see if it has the // expected contents with the given precision. -func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { +func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) { is := NewItems(items) oldErrors := fs.Stats.GetErrors() var objs []fs.Object + var dirs []*fs.Dir var err error var retries = *listRetries sleep := time.Second / 2 for i := 1; i <= retries; i++ { - objs, err = fs.NewLister().Start(f, "").GetObjects() + objs, dirs, err = fs.NewLister().Start(f, "").GetAll() if err != nil && err != fs.ErrorDirNotFound { t.Fatalf("Error listing: %v", err) } @@ -179,18 +181,29 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti if len(items) == 0 && oldErrors == 0 && fs.Stats.GetErrors() == 1 { fs.Stats.ResetErrors() } + // Check the directories - ignore if no directories returned + // for remotes which can't do directories + if expectedDirs != nil && len(dirs) != 0 { + actualDirs := []string{} + for _, dir := range dirs { + actualDirs = append(actualDirs, dir.Name) + } + sort.Strings(actualDirs) + sort.Strings(expectedDirs) + assert.Equal(t, expectedDirs, actualDirs, "directories") + } } // CheckListing checks the fs to see if it has the expected contents func CheckListing(t *testing.T, f fs.Fs, items []Item) { precision := f.Precision() - CheckListingWithPrecision(t, f, items, precision) + CheckListingWithPrecision(t, f, items, nil, precision) } // CheckItems checks the fs to see if it has only the items passed in // using a precision of fs.Config.ModifyWindow func CheckItems(t *testing.T, f fs.Fs, items ...Item) { - CheckListingWithPrecision(t, f, items, fs.Config.ModifyWindow) + CheckListingWithPrecision(t, f, items, nil, fs.Config.ModifyWindow) } // Time parses a time string or logs a fatal error @@ -300,7 +313,7 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, string, func(), error) // TestMkdir tests Mkdir works func TestMkdir(t *testing.T, remote fs.Fs) { - err := fs.Mkdir(remote) + err := fs.Mkdir(remote, "") require.NoError(t, err) CheckListing(t, remote, []Item{}) } @@ -314,6 +327,6 @@ func TestPurge(t *testing.T, remote fs.Fs) { // TestRmdir tests Rmdir works func TestRmdir(t *testing.T, remote fs.Fs) { - err := fs.Rmdir(remote) + err := fs.Rmdir(remote, "") require.NoError(t, err) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 660566639..efda48096 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -116,7 +116,7 @@ func TestFsRmdirEmpty(t *testing.T) { // TestFsRmdirNotFound tests deleting a non existent directory func TestFsRmdirNotFound(t *testing.T) { skipIfNotOk(t) - err := remote.Rmdir() + err := remote.Rmdir("") assert.Error(t, err, "Expecting error on Rmdir non existent") } @@ -127,6 +127,23 @@ func TestFsMkdir(t *testing.T) { fstest.TestMkdir(t, remote) } +// TestFsMkdirRmdirSubdir tests making and removing a sub directory +func TestFsMkdirRmdirSubdir(t *testing.T) { + skipIfNotOk(t) + dir := "dir/subdir" + err := fs.Mkdir(remote, dir) + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.Config.ModifyWindow) + + err = fs.Rmdir(remote, dir) + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{"dir"}, fs.Config.ModifyWindow) + + err = fs.Rmdir(remote, "dir") + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, fs.Config.ModifyWindow) +} + // TestFsListEmpty tests listing an empty directory func TestFsListEmpty(t *testing.T) { skipIfNotOk(t) @@ -501,7 +518,7 @@ func TestFsDirMove(t *testing.T) { // TestFsRmdirFull tests removing a non empty directory func TestFsRmdirFull(t *testing.T) { skipIfNotOk(t) - err := remote.Rmdir() + err := remote.Rmdir("") require.Error(t, err, "Expecting error on RMdir on non empty remote") } diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index fb87faca6..7e411c13f 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -448,7 +448,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the bucket if it doesn't exist -func (f *Fs) Mkdir() error { +func (f *Fs) Mkdir(dir string) error { + // Can't create subdirs + if dir != "" { + return nil + } _, err := f.svc.Buckets.Get(f.bucket).Do() if err == nil { // Bucket already exists @@ -470,8 +474,8 @@ func (f *Fs) Mkdir() error { // // Returns an error if it isn't empty: Error 409: The bucket you tried // to delete was not empty. -func (f *Fs) Rmdir() error { - if f.root != "" { +func (f *Fs) Rmdir(dir string) error { + if f.root != "" || dir != "" { return nil } return f.svc.Buckets.Delete(f.bucket).Do() diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index ef3431185..80aa3af52 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go index 0788a207f..1b97d5b7a 100644 --- a/hubic/hubic_test.go +++ b/hubic/hubic_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/local/local.go b/local/local.go index 375952ce4..67dbcc884 100644 --- a/local/local.go +++ b/local/local.go @@ -296,25 +296,28 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the directory if it doesn't exist -func (f *Fs) Mkdir() error { +func (f *Fs) Mkdir(dir string) error { // FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go - err := os.MkdirAll(f.root, 0777) + root := path.Join(f.root, dir) + err := os.MkdirAll(root, 0777) if err != nil { return err } - fi, err := os.Lstat(f.root) - if err != nil { - return err + if dir == "" { + fi, err := os.Lstat(root) + if err != nil { + return err + } + f.dev = readDevice(fi) } - f.dev = readDevice(fi) return nil } // Rmdir removes the directory // // If it isn't empty it will return an error -func (f *Fs) Rmdir() error { - return os.Remove(f.root) +func (f *Fs) Rmdir(dir string) error { + return os.Remove(path.Join(f.root, dir)) } // Precision of the file system diff --git a/local/local_test.go b/local/local_test.go index f197605ce..4dd685cf9 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index 35d2d2f34..744b3397e 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/url" + "path" "regexp" "strings" "time" @@ -452,8 +453,15 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { - return f.dirCache.FindRoot(true) +func (f *Fs) Mkdir(dir string) error { + err := f.dirCache.FindRoot(true) + if err != nil { + return err + } + if dir != "" { + _, err = f.dirCache.FindDir(dir, true) + } + return err } // deleteObject removes an object by ID @@ -471,17 +479,17 @@ func (f *Fs) deleteObject(id string) error { // purgeCheck removes the root directory, if check is set then it // refuses to do so if it has anything in -func (f *Fs) purgeCheck(check bool) error { - if f.root == "" { +func (f *Fs) purgeCheck(dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { return errors.New("can't purge root directory") } dc := f.dirCache - err := dc.FindRoot(false) + rootID, err := dc.FindDir(dir, false) if err != nil { return err } - rootID := dc.RootID() - item, _, err := f.readMetaDataForPath(f.root) + item, _, err := f.readMetaDataForPath(root) if err != nil { return err } @@ -495,7 +503,7 @@ func (f *Fs) purgeCheck(check bool) error { if err != nil { return err } - f.dirCache.ResetRoot() + f.dirCache.FlushDir(dir) if err != nil { return err } @@ -505,8 +513,8 @@ func (f *Fs) purgeCheck(check bool) error { // Rmdir deletes the root folder // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - return f.purgeCheck(true) +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) } // Precision return the precision of this Fs @@ -624,7 +632,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { // deleting all the files quicker than just running Remove() on the // result of List() func (f *Fs) Purge() error { - return f.purgeCheck(false) + return f.purgeCheck("", false) } // Hashes returns the supported hash sets. diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go index 01eeafb47..3b4e3fc8c 100644 --- a/onedrive/onedrive_test.go +++ b/onedrive/onedrive_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/s3/s3.go b/s3/s3.go index 94a1bad3b..1e166cd67 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -627,7 +627,11 @@ func (f *Fs) dirExists() (bool, error) { } // Mkdir creates the bucket if it doesn't exist -func (f *Fs) Mkdir() error { +func (f *Fs) Mkdir(dir string) error { + // Can't create subdirs + if dir != "" { + return nil + } exists, err := f.dirExists() if err != nil || exists { return err @@ -653,8 +657,8 @@ func (f *Fs) Mkdir() error { // Rmdir deletes the bucket if the fs is at the root // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - if f.root != "" { +func (f *Fs) Rmdir(dir string) error { + if f.root != "" || dir != "" { return nil } req := s3.DeleteBucketInput{ diff --git a/s3/s3_test.go b/s3/s3_test.go index 1ec896515..ca5998070 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/swift/swift.go b/swift/swift.go index 429a8c241..b0e3fd090 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -403,7 +403,11 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { +func (f *Fs) Mkdir(dir string) error { + // Can't create subdirs + if dir != "" { + return nil + } // Check to see if container exists first _, _, err := f.c.Container(f.container) if err == nil { @@ -419,8 +423,8 @@ func (f *Fs) Mkdir() error { // Rmdir deletes the container if the fs is at the root // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - if f.root != "" { +func (f *Fs) Rmdir(dir string) error { + if f.root != "" || dir != "" { return nil } return f.c.ContainerDelete(f.container) @@ -459,7 +463,7 @@ func (f *Fs) Purge() error { if err != nil { return err } - return f.Rmdir() + return f.Rmdir("") } // Copy src to this remote using server side copy operations. diff --git a/swift/swift_test.go b/swift/swift_test.go index d5a7edde9..f67a81b3c 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } diff --git a/yandex/yandex.go b/yandex/yandex.go index 67247fe8c..c2d062e79 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -390,25 +390,33 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { } // Mkdir creates the container if it doesn't exist -func (f *Fs) Mkdir() error { - return mkDirFullPath(f.yd, f.diskRoot) +func (f *Fs) Mkdir(dir string) error { + root := f.diskRoot + if dir != "" { + root += dir + "/" + } + return mkDirFullPath(f.yd, root) } // Rmdir deletes the container // // Returns an error if it isn't empty -func (f *Fs) Rmdir() error { - return f.purgeCheck(true) +func (f *Fs) Rmdir(dir string) error { + return f.purgeCheck(dir, true) } // purgeCheck remotes the root directory, if check is set then it // refuses to do so if it has anything in -func (f *Fs) purgeCheck(check bool) error { +func (f *Fs) purgeCheck(dir string, check bool) error { + root := f.diskRoot + if dir != "" { + root += dir + "/" + } if check { //to comply with rclone logic we check if the directory is empty before delete. //send request to get list of objects in this directory. var opt yandex.ResourceInfoRequestOptions - ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(f.diskRoot, opt).Exec() + ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(root, opt).Exec() if err != nil { return errors.Wrap(err, "rmdir failed") } @@ -417,7 +425,7 @@ func (f *Fs) purgeCheck(check bool) error { } } //delete directory - return f.yd.Delete(f.diskRoot, true) + return f.yd.Delete(root, true) } // Precision return the precision of this Fs @@ -431,7 +439,7 @@ func (f *Fs) Precision() time.Duration { // deleting all the files quicker than just running Remove() on the // result of List() func (f *Fs) Purge() error { - return f.purgeCheck(false) + return f.purgeCheck("", false) } // Hashes returns the supported hash sets. diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go index 5f7eba444..bb2f8dd34 100644 --- a/yandex/yandex_test.go +++ b/yandex/yandex_test.go @@ -23,6 +23,7 @@ func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }