From 0dd0d6a13e1e9f56d6f270de0bcd213e4319dec4 Mon Sep 17 00:00:00 2001 From: nielash Date: Tue, 11 Jul 2023 07:09:06 -0400 Subject: [PATCH] bisync: Add support for --create-empty-src-dirs - Fixes #6109 Sync creation and deletion of empty directories. https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed Also fixed an issue causing --resync to erroneously delete empty folders and duplicate files unique to Path2 https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs --- cmd/bisync/bisync_test.go | 2 + cmd/bisync/cmd.go | 4 +- cmd/bisync/deltas.go | 64 ++++++-- cmd/bisync/listing.go | 102 +++++++++++-- cmd/bisync/operations.go | 41 +++-- cmd/bisync/queue.go | 33 +++- ...testdir_path1.._testdir_path2.copy1to2.que | 1 + ..._testdir_path1.._testdir_path2.delete2.que | 1 + .../_testdir_path1.._testdir_path2.path1.lst | 7 + ...estdir_path1.._testdir_path2.path1.lst-new | 7 + .../_testdir_path1.._testdir_path2.path2.lst | 7 + ...estdir_path1.._testdir_path2.path2.lst-new | 7 + ..._path1.._testdir_path2.resync-copy2to1.que | 1 + .../test_createemptysrcdirs/golden/test.log | 142 ++++++++++++++++++ .../initial/RCLONE_TEST | 1 + .../initial/file1.copy1.txt | 0 .../initial/file1.copy2.txt | 0 .../initial/file1.copy3.txt | 0 .../initial/file1.copy4.txt | 0 .../initial/file1.copy5.txt | 0 .../test_createemptysrcdirs/initial/file1.txt | 0 .../modfiles/placeholder.txt | 0 .../test_createemptysrcdirs/scenario.txt | 87 +++++++++++ .../testdata/test_dry_run/golden/test.log | 3 - docs/content/bisync.md | 54 ++++++- 25 files changed, 512 insertions(+), 52 deletions(-) create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy1.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy2.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy3.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy4.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy5.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/modfiles/placeholder.txt create mode 100644 cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index 337f744ee..55e98a19e 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -614,6 +614,8 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) { opt.DryRun = true case "force": opt.Force = true + case "create-empty-src-dirs": + opt.CreateEmptySrcDirs = true case "remove-empty-dirs": opt.RemoveEmptyDirs = true case "check-sync-only": diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index cc8a2d925..1a4866035 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -31,6 +31,7 @@ type Options struct { CheckAccess bool CheckFilename string CheckSync CheckSyncMode + CreateEmptySrcDirs bool RemoveEmptyDirs bool MaxDelete int // percentage from 0 to 100 Force bool @@ -105,7 +106,8 @@ func init() { flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "") flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "") flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "") - flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove empty directories at the final cleanup step.", "") + flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "") + flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "") flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "") flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "") flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "") diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index bb7e56615..ccd43edbe 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -229,6 +229,24 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change ctxMove := b.opt.setDryRun(ctx) + // efficient isDir check + // we load the listing just once and store only the dirs + dirs1, dirs1Err := b.listDirsOnly(1) + if dirs1Err != nil { + b.critical = true + b.retryable = true + fs.Debugf(nil, "Error generating dirsonly list for path1: %v", dirs1Err) + return + } + + dirs2, dirs2Err := b.listDirsOnly(2) + if dirs2Err != nil { + b.critical = true + b.retryable = true + fs.Debugf(nil, "Error generating dirsonly list for path2: %v", dirs2Err) + return + } + // build a list of only the "deltaOther"s so we don't have to check more files than necessary // this is essentially the same as running rclone check with a --files-from filter, then exempting the --match results from being renamed // we therefore avoid having to list the same directory more than once. @@ -275,28 +293,32 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change b.indent("!WARNING", file, "New or changed in both paths") //if files are identical, leave them alone instead of renaming + if dirs1.has(file) && dirs2.has(file) { + fs.Debugf(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file) + } else { equal := matches.Has(file) if equal { fs.Infof(nil, "Files are equal! Skipping: %s", file) } else { fs.Debugf(nil, "Files are NOT equal: %s", file) - b.indent("!Path1", p1+"..path1", "Renaming Path1 copy") - if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil { - err = fmt.Errorf("path1 rename failed for %s: %w", p1, err) - b.critical = true - return - } - b.indent("!Path1", p2+"..path1", "Queue copy to Path2") - copy1to2.Add(file + "..path1") + b.indent("!Path1", p1+"..path1", "Renaming Path1 copy") + if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil { + err = fmt.Errorf("path1 rename failed for %s: %w", p1, err) + b.critical = true + return + } + b.indent("!Path1", p2+"..path1", "Queue copy to Path2") + copy1to2.Add(file + "..path1") - b.indent("!Path2", p2+"..path2", "Renaming Path2 copy") - if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil { - err = fmt.Errorf("path2 rename failed for %s: %w", file, err) - return - } - b.indent("!Path2", p1+"..path2", "Queue copy to Path1") - copy2to1.Add(file + "..path2") + b.indent("!Path2", p2+"..path2", "Renaming Path2 copy") + if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil { + err = fmt.Errorf("path2 rename failed for %s: %w", file, err) + return + } + b.indent("!Path2", p1+"..path2", "Queue copy to Path1") + copy2to1.Add(file + "..path2") } + } handled.Add(file) } } else { @@ -340,6 +362,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err != nil { return } + + //copy empty dirs from path2 to path1 (if --create-empty-src-dirs) + b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, "make") } if copy1to2.NotEmpty() { @@ -349,6 +374,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err != nil { return } + + //copy empty dirs from path1 to path2 (if --create-empty-src-dirs) + b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, "make") } if delete1.NotEmpty() { @@ -358,6 +386,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err != nil { return } + + //propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs) + b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove") } if delete2.NotEmpty() { @@ -367,6 +398,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err != nil { return } + + //propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs) + b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove") } return diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 520548bd7..b2edb7ec9 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -43,10 +43,11 @@ var tzLocal = false // fileInfo describes a file type fileInfo struct { - size int64 - time time.Time - hash string - id string + size int64 + time time.Time + hash string + id string + flags string } // fileList represents a listing @@ -76,17 +77,18 @@ func (ls *fileList) get(file string) *fileInfo { return ls.info[file] } -func (ls *fileList) put(file string, size int64, time time.Time, hash, id string) { +func (ls *fileList) put(file string, size int64, time time.Time, hash, id string, flags string) { fi := ls.get(file) if fi != nil { fi.size = size fi.time = time } else { fi = &fileInfo{ - size: size, - time: time, - hash: hash, - id: id, + size: size, + time: time, + hash: hash, + id: id, + flags: flags, } ls.info[file] = fi ls.list = append(ls.list, file) @@ -152,7 +154,11 @@ func (ls *fileList) save(ctx context.Context, listing string) error { id = "-" } - flags := "-" + flags := fi.flags + if flags == "" { + flags = "-" + } + _, err = fmt.Fprintf(file, lineFormat, flags, fi.size, hash, id, time, remote) if err != nil { _ = file.Close() @@ -217,7 +223,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) { } } - if flags != "-" || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil { + if (flags != "-" && flags != "d") || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil { fs.Logf(listing, "Ignoring incorrect line: %q", line) continue } @@ -229,7 +235,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) { } } - ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id) + ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id, flags) } return ls, nil @@ -262,7 +268,11 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l ls = newFileList() ls.hash = hashType var lock sync.Mutex - err = walk.ListR(ctx, f, "", false, depth, walk.ListObjects, func(entries fs.DirEntries) error { + listType := walk.ListObjects + if b.opt.CreateEmptySrcDirs { + listType = walk.ListAll + } + err = walk.ListR(ctx, f, "", false, depth, listType, func(entries fs.DirEntries) error { var firstErr error entries.ForObject(func(o fs.Object) { //tr := accounting.Stats(ctx).NewCheckingTransfer(o) // TODO @@ -277,12 +287,27 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l } } time := o.ModTime(ctx).In(TZ) - id := "" // TODO + id := "" // TODO + flags := "-" // "-" for a file and "d" for a directory lock.Lock() - ls.put(o.Remote(), o.Size(), time, hashVal, id) + ls.put(o.Remote(), o.Size(), time, hashVal, id, flags) lock.Unlock() //tr.Done(ctx, nil) // TODO }) + if b.opt.CreateEmptySrcDirs { + entries.ForDir(func(o fs.Directory) { + var ( + hashVal string + ) + time := o.ModTime(ctx).In(TZ) + id := "" // TODO + flags := "d" // "-" for a file and "d" for a directory + lock.Lock() + //record size as 0 instead of -1, so bisync doesn't think it's a google doc + ls.put(o.Remote(), 0, time, hashVal, id, flags) + lock.Unlock() + }) + } return firstErr }) if err == nil { @@ -304,3 +329,50 @@ func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error { b.retryable = true return fmt.Errorf("empty %s listing: %s", msg, listing) } + +// listingNum should be 1 for path1 or 2 for path2 +func (b *bisyncRun) loadListingNum(listingNum int) (*fileList, error) { + listingpath := b.basePath + ".path1.lst-new" + if listingNum == 2 { + listingpath = b.basePath + ".path2.lst-new" + } + + if b.opt.DryRun { + listingpath = strings.Replace(listingpath, ".lst-", ".lst-dry-", 1) + } + + fs.Debugf(nil, "loading listing for path %d at: %s", listingNum, listingpath) + return b.loadListing(listingpath) +} + +func (b *bisyncRun) listDirsOnly(listingNum int) (*fileList, error) { + var fulllisting *fileList + var dirsonly = newFileList() + var err error + + if !b.opt.CreateEmptySrcDirs { + return dirsonly, err + } + + fulllisting, err = b.loadListingNum(listingNum) + + if err != nil { + b.critical = true + b.retryable = true + fs.Debugf(nil, "Error loading listing to generate dirsonly list: %v", err) + return dirsonly, err + } + + for _, obj := range fulllisting.list { + info := fulllisting.get(obj) + + if info.flags == "d" { + fs.Debugf(nil, "found a dir: %s", obj) + dirsonly.put(obj, info.size, info.time, info.hash, info.id, info.flags) + } else { + fs.Debugf(nil, "not a dir: %s", obj) + } + } + + return dirsonly, err +} diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index 25c53cca2..250437e62 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -128,14 +128,14 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) { fs.Errorf(nil, "Bisync critical error: %v", err) fs.Errorf(nil, "Bisync aborted. Error is retryable without --resync due to --resilient mode.") } else { - if bilib.FileExists(listing1) { - _ = os.Rename(listing1, listing1+"-err") - } - if bilib.FileExists(listing2) { - _ = os.Rename(listing2, listing2+"-err") - } - fs.Errorf(nil, "Bisync critical error: %v", err) - fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.") + if bilib.FileExists(listing1) { + _ = os.Rename(listing1, listing1+"-err") + } + if bilib.FileExists(listing2) { + _ = os.Rename(listing2, listing2+"-err") + } + fs.Errorf(nil, "Bisync critical error: %v", err) + fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.") } return ErrBisyncAborted } @@ -413,11 +413,34 @@ func (b *bisyncRun) resync(octx, fctx context.Context, listing1, listing2 string // prevent overwriting Google Doc files (their size is -1) filterSync.Opt.MinSize = 0 } - if err = sync.Sync(ctxSync, b.fs2, b.fs1, false); err != nil { + if err = sync.CopyDir(ctxSync, b.fs2, b.fs1, b.opt.CreateEmptySrcDirs); err != nil { b.critical = true return err } + if b.opt.CreateEmptySrcDirs { + // copy Path2 back to Path1, for empty dirs + // the fastCopy above cannot include directories, because it relies on --files-from for filtering, + // so instead we'll copy them here, relying on fctx for our filtering. + + // This preserves the original resync order for backward compatibility. It is essentially: + // rclone copy Path2 Path1 --ignore-existing + // rclone copy Path1 Path2 --create-empty-src-dirs + // rclone copy Path2 Path1 --create-empty-src-dirs + + // although if we were starting from scratch, it might be cleaner and faster to just do: + // rclone copy Path2 Path1 --create-empty-src-dirs + // rclone copy Path1 Path2 --create-empty-src-dirs + + fs.Infof(nil, "Resynching Path2 to Path1 (for empty dirs)") + + //note copy (not sync) and dst comes before src + if err = sync.CopyDir(ctxSync, b.fs1, b.fs2, b.opt.CreateEmptySrcDirs); err != nil { + b.critical = true + return err + } + } + fs.Infof(nil, "Resync updating listings") if _, err = b.makeListing(fctx, b.fs1, listing1); err != nil { b.critical = true diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index 33db28f10..701c8a922 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -3,6 +3,7 @@ package bisync import ( "context" "fmt" + "sort" "github.com/rclone/rclone/cmd/bisync/bilib" "github.com/rclone/rclone/fs" @@ -23,7 +24,7 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. } } - return sync.CopyDir(ctxCopy, fdst, fsrc, false) + return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs) } func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error { @@ -60,6 +61,36 @@ func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, return err } +// operation should be "make" or "remove" +func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bilib.Names, dirsList *fileList, operation string) { + if b.opt.CreateEmptySrcDirs && (!b.opt.Resync || operation == "make") { + + candidatesList := candidates.ToList() + if operation == "remove" { + // reverse the sort order to ensure we remove subdirs before parent dirs + sort.Sort(sort.Reverse(sort.StringSlice(candidatesList))) + } + + for _, s := range candidatesList { + var direrr error + if dirsList.has(s) { //make sure it's a dir, not a file + if operation == "remove" { + //note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them. + direrr = operations.Rmdirs(ctx, dst, s, false) + } else if operation == "make" { + direrr = operations.Mkdir(ctx, dst, s) + } else { + direrr = fmt.Errorf("invalid operation. Expected 'make' or 'remove', received '%q'", operation) + } + + if direrr != nil { + fs.Debugf(nil, "Error syncing directory: %v", direrr) + } + } + } + } +} + func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error { if !b.opt.SaveQueues { return nil diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que new file mode 100644 index 000000000..4f985e416 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que @@ -0,0 +1 @@ +"subdir" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que new file mode 100644 index 000000000..4f985e416 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que @@ -0,0 +1 @@ +"subdir" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst new file mode 100644 index 000000000..5d46d4069 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst @@ -0,0 +1,7 @@ +# bisync listing v1 from test +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new new file mode 100644 index 000000000..5d46d4069 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -0,0 +1,7 @@ +# bisync listing v1 from test +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst new file mode 100644 index 000000000..5d46d4069 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst @@ -0,0 +1,7 @@ +# bisync listing v1 from test +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new new file mode 100644 index 000000000..5d46d4069 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -0,0 +1,7 @@ +# bisync listing v1 from test +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt" +- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que new file mode 100644 index 000000000..4f985e416 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que @@ -0,0 +1 @@ +"subdir" diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log new file mode 100644 index 000000000..b8799524a --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log @@ -0,0 +1,142 @@ +(01) : test createemptysrcdirs + + +(02) : test initial bisync +(03) : touch-glob 2001-01-02 {datadir/} placeholder.txt +(04) : copy-as {datadir/}placeholder.txt {path1/} file1.txt +(05) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +(06) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +(07) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +(08) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +(09) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt +(10) : bisync resync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : Resynching Path1 to Path2 +INFO : Resync updating listings +INFO : Bisync successful + +(11) : test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder +(12) : copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt +(13) : touch-glob 2001-01-02 {path1/} subdir +(14) : delete-file {path1/}subdir/placeholder.txt + +(15) : test 2. Run bisync without --create-empty-src-dirs +(16) : bisync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Path1 checking for diffs +INFO : Path2 checking for diffs +INFO : No changes found +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(17) : test 3. Confirm the subdir exists only on Path1 and not Path2 +(18) : list-dirs {path1/} +subdir/ +(19) : list-dirs {path2/} + +(20) : test 4.Run bisync WITH --create-empty-src-dirs +(21) : bisync create-empty-src-dirs +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Path1 checking for diffs +INFO : - Path1 File is new - subdir +INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted +INFO : Path2 checking for diffs +INFO : Applying changes +INFO : - Path1 Queue copy to Path2 - {path2/}subdir +INFO : - Path1 Do queued copies to - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(22) : test 5. Confirm the subdir exists on both paths +(23) : list-dirs {path1/} +subdir/ +(24) : list-dirs {path2/} +subdir/ + +(25) : test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty) +(26) : purge-children {path1/} +(27) : copy-as {datadir/}placeholder.txt {path1/} file1.txt +(28) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +(29) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +(30) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +(31) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +(32) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt + +(33) : test 7. Run bisync without --create-empty-src-dirs +(34) : bisync +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Path1 checking for diffs +INFO : - Path1 File was deleted - RCLONE_TEST +INFO : - Path1 File was deleted - subdir +INFO : Path1: 2 changes: 0 new, 0 newer, 0 older, 2 deleted +INFO : Path2 checking for diffs +INFO : - Path2 File was deleted - subdir +INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted +INFO : Applying changes +INFO : - Path2 Queue delete - {path2/}RCLONE_TEST +INFO : - Do queued deletes on - Path2 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(35) : test 8. Confirm the subdir exists only on Path2 and not Path1 +(36) : list-dirs {path1/} +(37) : list-dirs {path2/} +subdir/ + +(38) : test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs +(39) : bisync resync create-empty-src-dirs +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying unique Path2 files to Path1 +INFO : - Path2 Resync will copy to Path1 - subdir +INFO : - Path2 Resync is doing queued copies to - Path1 +INFO : Resynching Path1 to Path2 +INFO : Resynching Path2 to Path1 (for empty dirs) +INFO : Resync updating listings +INFO : Bisync successful +(40) : list-dirs {path1/} +subdir/ +(41) : list-dirs {path2/} +subdir/ + +(42) : purge-children {path1/} +(43) : copy-as {datadir/}placeholder.txt {path1/} file1.txt +(44) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +(45) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +(46) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +(47) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +(48) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt +(49) : list-dirs {path1/} +(50) : list-dirs {path2/} +subdir/ + +(51) : bisync create-empty-src-dirs +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Path1 checking for diffs +INFO : - Path1 File was deleted - subdir +INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted +INFO : Path2 checking for diffs +INFO : Applying changes +INFO : - Path2 Queue delete - {path2/}subdir +INFO : - Do queued deletes on - Path2 +INFO : subdir: Removing directory +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(52) : test 10. Confirm the subdir has been removed on both paths +(53) : list-dirs {path1/} +(54) : list-dirs {path2/} + +(55) : test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime) +(56) : bisync create-empty-src-dirs +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Path1 checking for diffs +INFO : Path2 checking for diffs +INFO : No changes found +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST b/cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST new file mode 100644 index 000000000..d8ca97c2a --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST @@ -0,0 +1 @@ +This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete. diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy1.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy2.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy3.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy3.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy4.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy4.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy5.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy5.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.txt b/cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/modfiles/placeholder.txt b/cmd/bisync/testdata/test_createemptysrcdirs/modfiles/placeholder.txt new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt b/cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt new file mode 100644 index 000000000..8b39ae2b5 --- /dev/null +++ b/cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt @@ -0,0 +1,87 @@ +test createemptysrcdirs +# Test the --create-empty-src-dirs logic. +# Should behave the same way as rclone sync. +# Without this flag, empty directories created/deleted on one side are NOT created/deleted on the other side +# With this flag, empty directories created/deleted on one side are created/deleted on the other side; the result should be an exact mirror. +# +# Placeholders are necessary to ensure that git does not lose our empty folders +# After the initial setup sync: +# 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder +# 2. Run bisync without --create-empty-src-dirs +# 3. Confirm the subdir exists only on Path1 and not Path2 +# 4. Run bisync WITH --create-empty-src-dirs +# 5. Confirm the subdir exists on both paths +# 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty) +# 7. Run bisync without --create-empty-src-dirs +# 8. Confirm the subdir exists only on Path2 and not Path1 +# 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs +# 10. Confirm the subdir has been removed on both paths + +test initial bisync +touch-glob 2001-01-02 {datadir/} placeholder.txt +copy-as {datadir/}placeholder.txt {path1/} file1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt +bisync resync + +test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder +copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt +touch-glob 2001-01-02 {path1/} subdir +delete-file {path1/}subdir/placeholder.txt + +test 2. Run bisync without --create-empty-src-dirs +bisync + +test 3. Confirm the subdir exists only on Path1 and not Path2 +list-dirs {path1/} +list-dirs {path2/} + +test 4.Run bisync WITH --create-empty-src-dirs +bisync create-empty-src-dirs + +test 5. Confirm the subdir exists on both paths +list-dirs {path1/} +list-dirs {path2/} + +test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty) +purge-children {path1/} +copy-as {datadir/}placeholder.txt {path1/} file1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt + +test 7. Run bisync without --create-empty-src-dirs +bisync + +test 8. Confirm the subdir exists only on Path2 and not Path1 +list-dirs {path1/} +list-dirs {path2/} + +test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs +bisync resync create-empty-src-dirs +list-dirs {path1/} +list-dirs {path2/} + +purge-children {path1/} +copy-as {datadir/}placeholder.txt {path1/} file1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt +copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt +list-dirs {path1/} +list-dirs {path2/} + +bisync create-empty-src-dirs + +test 10. Confirm the subdir has been removed on both paths +list-dirs {path1/} +list-dirs {path2/} + +test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime) +bisync create-empty-src-dirs \ No newline at end of file diff --git a/cmd/bisync/testdata/test_dry_run/golden/test.log b/cmd/bisync/testdata/test_dry_run/golden/test.log index cb3f8d263..13bd9654a 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/test.log +++ b/cmd/bisync/testdata/test_dry_run/golden/test.log @@ -54,13 +54,10 @@ NOTICE: file4.txt: Skipped copy as --dry-run is set (size 0) NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19) INFO : Resynching Path1 to Path2 NOTICE: file1.txt: Skipped copy as --dry-run is set (size 0) -NOTICE: file10.txt: Skipped delete as --dry-run is set (size 19) NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19) NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13) NOTICE: file3.txt: Skipped copy as --dry-run is set (size 0) -NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0) NOTICE: file5.txt: Skipped copy (or update modification time) as --dry-run is set (size 39) -NOTICE: file6.txt: Skipped delete as --dry-run is set (size 19) NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19) INFO : Resync updating listings INFO : Bisync successful diff --git a/docs/content/bisync.md b/docs/content/bisync.md index a3a714d9b..dc0256d30 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -91,6 +91,8 @@ Optional Flags: If exceeded, the bisync run will abort. (default: 50%) --force Bypass `--max-delete` safety check and run the sync. Consider using with `--verbose` + --create-empty-src-dirs Sync creation and deletion of empty directories. + (Not compatible with --remove-empty-dirs) --remove-empty-dirs Remove empty directories at the final cleanup step. -1, --resync Performs the resync run. Warning: Path1 files may overwrite Path2 versions. @@ -125,7 +127,7 @@ Cloud references are distinguished by having a `:` in the argument (see [Windows support](#windows) below). Path1 and Path2 are treated equally, in that neither has priority for -file changes, and access efficiency does not change whether a remote +file changes (except during [`--resync`](#resync)), and access efficiency does not change whether a remote is on Path1 or Path2. The listings in bisync working directory (default: `~/.cache/rclone/bisync`) @@ -134,8 +136,8 @@ to individual directories within the tree may be set up, e.g.: `path_to_local_tree..dropbox_subdir.lst`. Any empty directories after the sync on both the Path1 and Path2 -filesystems are not deleted by default. If the `--remove-empty-dirs` -flag is specified, then both paths will have any empty directories purged +filesystems are not deleted by default, unless `--create-empty-src-dirs` is specified. +If the `--remove-empty-dirs` flag is specified, then both paths will have ALL empty directories purged as the last step in the process. ## Command-line flags @@ -144,15 +146,31 @@ as the last step in the process. This will effectively make both Path1 and Path2 filesystems contain a matching superset of all files. Path2 files that do not exist in Path1 will -be copied to Path1, and the process will then sync the Path1 tree to Path2. +be copied to Path1, and the process will then copy the Path1 tree to Path2. -The base directories on the both Path1 and Path2 filesystems must exist +The `--resync` sequence is roughly equivalent to: +``` +rclone copy Path2 Path1 --ignore-existing +rclone copy Path1 Path2 +``` +Or, if using `--create-empty-src-dirs`: +``` +rclone copy Path2 Path1 --ignore-existing +rclone copy Path1 Path2 --create-empty-src-dirs +rclone copy Path2 Path1 --create-empty-src-dirs +``` + +The base directories on both Path1 and Path2 filesystems must exist or bisync will fail. This is required for safety - that bisync can verify that both paths are valid. -When using `--resync`, a newer version of a file either on Path1 or Path2 -filesystem, will overwrite the file on the other path (only the last version -will be kept). Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags). +When using `--resync`, a newer version of a file on the Path2 filesystem +will be overwritten by the Path1 filesystem version. +(Note that this is [NOT entirely symmetrical](https://github.com/rclone/rclone/issues/5681#issuecomment-938761815).) +Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags). + +[//]: # (I reverted a recent change in the above paragraph, as it was incorrect. +https://github.com/rclone/rclone/commit/dd72aff98a46c6e20848ac7ae5f7b19d45802493 ) For a resync run, one of the paths may be empty (no files in the path tree). The resync run should result in files on both paths, else a normal non-resync @@ -493,6 +511,22 @@ rclone copy PATH1 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs rclone copy PATH2 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs ``` +### Empty directories + +By default, new/deleted empty directories on one path are _not_ propagated to the other side. +This is because bisync (and rclone) natively works on files, not directories. +However, this can be changed with the `--create-empty-src-dirs` flag, which works in +much the same way as in [`sync`](/commands/rclone_sync/) and [`copy`](/commands/rclone_copy/). +When used, empty directories created or deleted on one side will also be created or deleted on the other side. +The following should be noted: +* `--create-empty-src-dirs` is not compatible with `--remove-empty-dirs`. Use only one or the other (or neither). +* It is not recommended to switch back and forth between `--create-empty-src-dirs` +and the default (no `--create-empty-src-dirs`) without running `--resync`. +This is because it may appear as though all directories (not just the empty ones) were created/deleted, +when actually you've just toggled between making them visible/invisible to bisync. +It looks scarier than it is, but it's still probably best to stick to one or the other, +and use `--resync` when you need to switch. + ### Renamed directories Renaming a folder on the Path1 side results in deleting all files on @@ -1187,11 +1221,15 @@ about _Unison_ and synchronization in general. ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry) causing dry runs to inadvertently commit filter changes +* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs) +causing `--resync` to erroneously delete empty folders and duplicate files unique to Path2 * `--check-access` is now enforced during `--resync`, preventing data loss in [certain user error scenarios](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=%2D%2Dcheck%2Daccess%20doesn%27t%20always%20fail%20when%20it%20should) * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=5.%20Bisync%20reads%20files%20in%20excluded%20directories%20during%20delete%20operations) causing bisync to consider more files than necessary due to overbroad filters during delete operations * [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides) (identical files are now left alone instead of renamed) +* Added [support for `--create-empty-src-dirs`](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed) * Added experimental `--resilient` mode to allow [recovery from self-correctable errors](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20Bisync%20should%20be%20more%20resilient%20to%20self%2Dcorrectable%20errors) * Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes) to distinguish from `--ignore-checksum` +* [Performance improvements](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20Deletes%20take%20several%20times%20longer%20than%20copies) for large remotes