github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-05 01:42:31 +01:00
Code Releases Activity

bisync: Add support for --create-empty-src-dirs - Fixes #6109

Browse Source 
Sync creation and deletion of empty directories.
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed

Also fixed an issue causing --resync to erroneously delete empty folders and duplicate files unique to Path2
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs
This commit is contained in:
nielash 2023-07-11 07:09:06 -04:00 committed by Nick Craig-Wood
parent e5bde42303
commit 0dd0d6a13e
25 changed files with 512 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/bisync/bisync_test.go
View File

@ -614,6 +614,8 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
opt.DryRun = true
case "force":
opt.Force = true
case "create-empty-src-dirs":
opt.CreateEmptySrcDirs = true
case "remove-empty-dirs":
opt.RemoveEmptyDirs = true
case "check-sync-only":

4
cmd/bisync/cmd.go
View File

@ -31,6 +31,7 @@ type Options struct {
CheckAccess bool
CheckFilename string
CheckSync CheckSyncMode
CreateEmptySrcDirs bool
RemoveEmptyDirs bool
MaxDelete int // percentage from 0 to 100
Force bool
@ -105,7 +106,8 @@ func init() {
flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove empty directories at the final cleanup step.", "")
flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")
flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")

64
cmd/bisync/deltas.go
View File

@ -229,6 +229,24 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
ctxMove := b.opt.setDryRun(ctx)
// efficient isDir check
// we load the listing just once and store only the dirs
dirs1, dirs1Err := b.listDirsOnly(1)
if dirs1Err != nil {
b.critical = true
b.retryable = true
fs.Debugf(nil, "Error generating dirsonly list for path1: %v", dirs1Err)
return
}
dirs2, dirs2Err := b.listDirsOnly(2)
if dirs2Err != nil {
b.critical = true
b.retryable = true
fs.Debugf(nil, "Error generating dirsonly list for path2: %v", dirs2Err)
return
}
// build a list of only the "deltaOther"s so we don't have to check more files than necessary
// this is essentially the same as running rclone check with a --files-from filter, then exempting the --match results from being renamed
// we therefore avoid having to list the same directory more than once.
@ -275,28 +293,32 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
b.indent("!WARNING", file, "New or changed in both paths")
//if files are identical, leave them alone instead of renaming
if dirs1.has(file) && dirs2.has(file) {
fs.Debugf(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
} else {
equal := matches.Has(file)
if equal {
fs.Infof(nil, "Files are equal! Skipping: %s", file)
} else {
fs.Debugf(nil, "Files are NOT equal: %s", file)
b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")
if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil {
err = fmt.Errorf("path1 rename failed for %s: %w", p1, err)
b.critical = true
return
}
b.indent("!Path1", p2+"..path1", "Queue copy to Path2")
copy1to2.Add(file + "..path1")
b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")
if err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil {
err = fmt.Errorf("path1 rename failed for %s: %w", p1, err)
b.critical = true
return
}
b.indent("!Path1", p2+"..path1", "Queue copy to Path2")
copy1to2.Add(file + "..path1")
b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")
if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil {
err = fmt.Errorf("path2 rename failed for %s: %w", file, err)
return
}
b.indent("!Path2", p1+"..path2", "Queue copy to Path1")
copy2to1.Add(file + "..path2")
b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")
if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil {
err = fmt.Errorf("path2 rename failed for %s: %w", file, err)
return
}
b.indent("!Path2", p1+"..path2", "Queue copy to Path1")
copy2to1.Add(file + "..path2")
}
}
handled.Add(file)
}
} else {
@ -340,6 +362,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if err != nil {
return
}
//copy empty dirs from path2 to path1 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, "make")
}
if copy1to2.NotEmpty() {
@ -349,6 +374,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if err != nil {
return
}
//copy empty dirs from path1 to path2 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, "make")
}
if delete1.NotEmpty() {
@ -358,6 +386,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if err != nil {
return
}
//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")
}
if delete2.NotEmpty() {
@ -367,6 +398,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if err != nil {
return
}
//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")
}
return

102
cmd/bisync/listing.go
View File

@ -43,10 +43,11 @@ var tzLocal = false
// fileInfo describes a file
type fileInfo struct {
size int64
time time.Time
hash string
id string
size int64
time time.Time
hash string
id string
flags string
}
// fileList represents a listing
@ -76,17 +77,18 @@ func (ls *fileList) get(file string) *fileInfo {
return ls.info[file]
}
func (ls *fileList) put(file string, size int64, time time.Time, hash, id string) {
func (ls *fileList) put(file string, size int64, time time.Time, hash, id string, flags string) {
fi := ls.get(file)
if fi != nil {
fi.size = size
fi.time = time
} else {
fi = &fileInfo{
size: size,
time: time,
hash: hash,
id: id,
size: size,
time: time,
hash: hash,
id: id,
flags: flags,
}
ls.info[file] = fi
ls.list = append(ls.list, file)
@ -152,7 +154,11 @@ func (ls *fileList) save(ctx context.Context, listing string) error {
id = "-"
}
flags := "-"
flags := fi.flags
if flags == "" {
flags = "-"
}
_, err = fmt.Fprintf(file, lineFormat, flags, fi.size, hash, id, time, remote)
if err != nil {
_ = file.Close()
@ -217,7 +223,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) {
}
}
if flags != "-" || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil {
if (flags != "-" && flags != "d") || id != "-" || sizeErr != nil || timeErr != nil || hashErr != nil || nameErr != nil {
fs.Logf(listing, "Ignoring incorrect line: %q", line)
continue
}
@ -229,7 +235,7 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) {
}
}
ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id)
ls.put(nameVal, sizeVal, timeVal.In(TZ), hashVal, id, flags)
}
return ls, nil
@ -262,7 +268,11 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
ls = newFileList()
ls.hash = hashType
var lock sync.Mutex
err = walk.ListR(ctx, f, "", false, depth, walk.ListObjects, func(entries fs.DirEntries) error {
listType := walk.ListObjects
if b.opt.CreateEmptySrcDirs {
listType = walk.ListAll
}
err = walk.ListR(ctx, f, "", false, depth, listType, func(entries fs.DirEntries) error {
var firstErr error
entries.ForObject(func(o fs.Object) {
//tr := accounting.Stats(ctx).NewCheckingTransfer(o) // TODO
@ -277,12 +287,27 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l
}
}
time := o.ModTime(ctx).In(TZ)
id := "" // TODO
id := "" // TODO
flags := "-" // "-" for a file and "d" for a directory
lock.Lock()
ls.put(o.Remote(), o.Size(), time, hashVal, id)
ls.put(o.Remote(), o.Size(), time, hashVal, id, flags)
lock.Unlock()
//tr.Done(ctx, nil) // TODO
})
if b.opt.CreateEmptySrcDirs {
entries.ForDir(func(o fs.Directory) {
var (
hashVal string
)
time := o.ModTime(ctx).In(TZ)
id := "" // TODO
flags := "d" // "-" for a file and "d" for a directory
lock.Lock()
//record size as 0 instead of -1, so bisync doesn't think it's a google doc
ls.put(o.Remote(), 0, time, hashVal, id, flags)
lock.Unlock()
})
}
return firstErr
})
if err == nil {
@ -304,3 +329,50 @@ func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error {
b.retryable = true
return fmt.Errorf("empty %s listing: %s", msg, listing)
}
// listingNum should be 1 for path1 or 2 for path2
func (b *bisyncRun) loadListingNum(listingNum int) (*fileList, error) {
listingpath := b.basePath + ".path1.lst-new"
if listingNum == 2 {
listingpath = b.basePath + ".path2.lst-new"
}
if b.opt.DryRun {
listingpath = strings.Replace(listingpath, ".lst-", ".lst-dry-", 1)
}
fs.Debugf(nil, "loading listing for path %d at: %s", listingNum, listingpath)
return b.loadListing(listingpath)
}
func (b *bisyncRun) listDirsOnly(listingNum int) (*fileList, error) {
var fulllisting *fileList
var dirsonly = newFileList()
var err error
if !b.opt.CreateEmptySrcDirs {
return dirsonly, err
}
fulllisting, err = b.loadListingNum(listingNum)
if err != nil {
b.critical = true
b.retryable = true
fs.Debugf(nil, "Error loading listing to generate dirsonly list: %v", err)
return dirsonly, err
}
for _, obj := range fulllisting.list {
info := fulllisting.get(obj)
if info.flags == "d" {
fs.Debugf(nil, "found a dir: %s", obj)
dirsonly.put(obj, info.size, info.time, info.hash, info.id, info.flags)
} else {
fs.Debugf(nil, "not a dir: %s", obj)
}
}
return dirsonly, err
}

41
cmd/bisync/operations.go
View File

@ -128,14 +128,14 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
fs.Errorf(nil, "Bisync critical error: %v", err)
fs.Errorf(nil, "Bisync aborted. Error is retryable without --resync due to --resilient mode.")
} else {
if bilib.FileExists(listing1) {
_ = os.Rename(listing1, listing1+"-err")
}
if bilib.FileExists(listing2) {
_ = os.Rename(listing2, listing2+"-err")
}
fs.Errorf(nil, "Bisync critical error: %v", err)
fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.")
if bilib.FileExists(listing1) {
_ = os.Rename(listing1, listing1+"-err")
}
if bilib.FileExists(listing2) {
_ = os.Rename(listing2, listing2+"-err")
}
fs.Errorf(nil, "Bisync critical error: %v", err)
fs.Errorf(nil, "Bisync aborted. Must run --resync to recover.")
}
return ErrBisyncAborted
}
@ -413,11 +413,34 @@ func (b *bisyncRun) resync(octx, fctx context.Context, listing1, listing2 string
// prevent overwriting Google Doc files (their size is -1)
filterSync.Opt.MinSize = 0
}
if err = sync.Sync(ctxSync, b.fs2, b.fs1, false); err != nil {
if err = sync.CopyDir(ctxSync, b.fs2, b.fs1, b.opt.CreateEmptySrcDirs); err != nil {
b.critical = true
return err
}
if b.opt.CreateEmptySrcDirs {
// copy Path2 back to Path1, for empty dirs
// the fastCopy above cannot include directories, because it relies on --files-from for filtering,
// so instead we'll copy them here, relying on fctx for our filtering.
// This preserves the original resync order for backward compatibility. It is essentially:
// rclone copy Path2 Path1 --ignore-existing
// rclone copy Path1 Path2 --create-empty-src-dirs
// rclone copy Path2 Path1 --create-empty-src-dirs
// although if we were starting from scratch, it might be cleaner and faster to just do:
// rclone copy Path2 Path1 --create-empty-src-dirs
// rclone copy Path1 Path2 --create-empty-src-dirs
fs.Infof(nil, "Resynching Path2 to Path1 (for empty dirs)")
//note copy (not sync) and dst comes before src
if err = sync.CopyDir(ctxSync, b.fs1, b.fs2, b.opt.CreateEmptySrcDirs); err != nil {
b.critical = true
return err
}
}
fs.Infof(nil, "Resync updating listings")
if _, err = b.makeListing(fctx, b.fs1, listing1); err != nil {
b.critical = true

33
cmd/bisync/queue.go
View File

@ -3,6 +3,7 @@ package bisync
import (
"context"
"fmt"
"sort"
"github.com/rclone/rclone/cmd/bisync/bilib"
"github.com/rclone/rclone/fs"
@ -23,7 +24,7 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
}
}
return sync.CopyDir(ctxCopy, fdst, fsrc, false)
return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
}
func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {
@ -60,6 +61,36 @@ func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names,
return err
}
// operation should be "make" or "remove"
func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bilib.Names, dirsList *fileList, operation string) {
if b.opt.CreateEmptySrcDirs && (!b.opt.Resync || operation == "make") {
candidatesList := candidates.ToList()
if operation == "remove" {
// reverse the sort order to ensure we remove subdirs before parent dirs
sort.Sort(sort.Reverse(sort.StringSlice(candidatesList)))
}
for _, s := range candidatesList {
var direrr error
if dirsList.has(s) { //make sure it's a dir, not a file
if operation == "remove" {
//note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.
direrr = operations.Rmdirs(ctx, dst, s, false)
} else if operation == "make" {
direrr = operations.Mkdir(ctx, dst, s)
} else {
direrr = fmt.Errorf("invalid operation. Expected 'make' or 'remove', received '%q'", operation)
}
if direrr != nil {
fs.Debugf(nil, "Error syncing directory: %v", direrr)
}
}
}
}
}
func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
if !b.opt.SaveQueues {
return nil

1
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored Normal file
View File

@ -0,0 +1 @@
"subdir"

1
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.delete2.que vendored Normal file
View File

@ -0,0 +1 @@
"subdir"

7
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst vendored Normal file
View File

@ -0,0 +1,7 @@
# bisync listing v1 from test
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

7
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path1.lst-new vendored Normal file
View File

@ -0,0 +1,7 @@
# bisync listing v1 from test
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

7
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst vendored Normal file
View File

@ -0,0 +1,7 @@
# bisync listing v1 from test
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

7
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.path2.lst-new vendored Normal file
View File

@ -0,0 +1,7 @@
# bisync listing v1 from test
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2001-01-02T00:00:00.000000000+0000 "file1.txt"

1
cmd/bisync/testdata/test_createemptysrcdirs/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que vendored Normal file
View File

@ -0,0 +1 @@
"subdir"

142
cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log vendored Normal file
View File

@ -0,0 +1,142 @@
(01) : test createemptysrcdirs
(02) : test initial bisync
(03) : touch-glob 2001-01-02 {datadir/} placeholder.txt
(04) : copy-as {datadir/}placeholder.txt {path1/} file1.txt
(05) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
(06) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
(07) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
(08) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
(09) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
(10) : bisync resync
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying unique Path2 files to Path1
INFO : Resynching Path1 to Path2
INFO : Resync updating listings
INFO : Bisync successful
(11) : test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
(12) : copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt
(13) : touch-glob 2001-01-02 {path1/} subdir
(14) : delete-file {path1/}subdir/placeholder.txt
(15) : test 2. Run bisync without --create-empty-src-dirs
(16) : bisync
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Path1 checking for diffs
INFO : Path2 checking for diffs
INFO : No changes found
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(17) : test 3. Confirm the subdir exists only on Path1 and not Path2
(18) : list-dirs {path1/}
subdir/
(19) : list-dirs {path2/}
(20) : test 4.Run bisync WITH --create-empty-src-dirs
(21) : bisync create-empty-src-dirs
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Path1 checking for diffs
INFO : - Path1 File is new - subdir
INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}subdir
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(22) : test 5. Confirm the subdir exists on both paths
(23) : list-dirs {path1/}
subdir/
(24) : list-dirs {path2/}
subdir/
(25) : test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
(26) : purge-children {path1/}
(27) : copy-as {datadir/}placeholder.txt {path1/} file1.txt
(28) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
(29) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
(30) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
(31) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
(32) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
(33) : test 7. Run bisync without --create-empty-src-dirs
(34) : bisync
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Path1 checking for diffs
INFO : - Path1 File was deleted - RCLONE_TEST
INFO : - Path1 File was deleted - subdir
INFO : Path1: 2 changes: 0 new, 0 newer, 0 older, 2 deleted
INFO : Path2 checking for diffs
INFO : - Path2 File was deleted - subdir
INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}RCLONE_TEST
INFO : - Do queued deletes on - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(35) : test 8. Confirm the subdir exists only on Path2 and not Path1
(36) : list-dirs {path1/}
(37) : list-dirs {path2/}
subdir/
(38) : test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
(39) : bisync resync create-empty-src-dirs
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying unique Path2 files to Path1
INFO : - Path2 Resync will copy to Path1 - subdir
INFO : - Path2 Resync is doing queued copies to - Path1
INFO : Resynching Path1 to Path2
INFO : Resynching Path2 to Path1 (for empty dirs)
INFO : Resync updating listings
INFO : Bisync successful
(40) : list-dirs {path1/}
subdir/
(41) : list-dirs {path2/}
subdir/
(42) : purge-children {path1/}
(43) : copy-as {datadir/}placeholder.txt {path1/} file1.txt
(44) : copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
(45) : copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
(46) : copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
(47) : copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
(48) : copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
(49) : list-dirs {path1/}
(50) : list-dirs {path2/}
subdir/
(51) : bisync create-empty-src-dirs
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Path1 checking for diffs
INFO : - Path1 File was deleted - subdir
INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir
INFO : - Do queued deletes on - Path2
INFO : subdir: Removing directory
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(52) : test 10. Confirm the subdir has been removed on both paths
(53) : list-dirs {path1/}
(54) : list-dirs {path2/}
(55) : test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime)
(56) : bisync create-empty-src-dirs
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Path1 checking for diffs
INFO : Path2 checking for diffs
INFO : No changes found
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

1
cmd/bisync/testdata/test_createemptysrcdirs/initial/RCLONE_TEST vendored Normal file
View File

@ -0,0 +1 @@
This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete.

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy1.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy2.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy3.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy4.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.copy5.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/initial/file1.txt vendored Normal file
View File

0
cmd/bisync/testdata/test_createemptysrcdirs/modfiles/placeholder.txt vendored Normal file
View File

87
cmd/bisync/testdata/test_createemptysrcdirs/scenario.txt vendored Normal file
View File

@ -0,0 +1,87 @@
test createemptysrcdirs
# Test the --create-empty-src-dirs logic.
# Should behave the same way as rclone sync.
# Without this flag, empty directories created/deleted on one side are NOT created/deleted on the other side
# With this flag, empty directories created/deleted on one side are created/deleted on the other side; the result should be an exact mirror.
#
# Placeholders are necessary to ensure that git does not lose our empty folders
# After the initial setup sync:
# 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
# 2. Run bisync without --create-empty-src-dirs
# 3. Confirm the subdir exists only on Path1 and not Path2
# 4. Run bisync WITH --create-empty-src-dirs
# 5. Confirm the subdir exists on both paths
# 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
# 7. Run bisync without --create-empty-src-dirs
# 8. Confirm the subdir exists only on Path2 and not Path1
# 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
# 10. Confirm the subdir has been removed on both paths
test initial bisync
touch-glob 2001-01-02 {datadir/} placeholder.txt
copy-as {datadir/}placeholder.txt {path1/} file1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
bisync resync
test 1. Create an empty dir on Path1 by creating subdir/placeholder.txt and then deleting the placeholder
copy-as {datadir/}placeholder.txt {path1/} subdir/placeholder.txt
touch-glob 2001-01-02 {path1/} subdir
delete-file {path1/}subdir/placeholder.txt
test 2. Run bisync without --create-empty-src-dirs
bisync
test 3. Confirm the subdir exists only on Path1 and not Path2
list-dirs {path1/}
list-dirs {path2/}
test 4.Run bisync WITH --create-empty-src-dirs
bisync create-empty-src-dirs
test 5. Confirm the subdir exists on both paths
list-dirs {path1/}
list-dirs {path2/}
test 6. Delete the empty dir on Path1 using purge-children (and also add files so the path isn't empty)
purge-children {path1/}
copy-as {datadir/}placeholder.txt {path1/} file1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
test 7. Run bisync without --create-empty-src-dirs
bisync
test 8. Confirm the subdir exists only on Path2 and not Path1
list-dirs {path1/}
list-dirs {path2/}
test 9. Reset, do the delete again, and run bisync WITH --create-empty-src-dirs
bisync resync create-empty-src-dirs
list-dirs {path1/}
list-dirs {path2/}
purge-children {path1/}
copy-as {datadir/}placeholder.txt {path1/} file1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy1.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy2.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy3.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy4.txt
copy-as {datadir/}placeholder.txt {path1/} file1.copy5.txt
list-dirs {path1/}
list-dirs {path2/}
bisync create-empty-src-dirs
test 10. Confirm the subdir has been removed on both paths
list-dirs {path1/}
list-dirs {path2/}
test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime)
bisync create-empty-src-dirs

3
cmd/bisync/testdata/test_dry_run/golden/test.log vendored
View File

@ -54,13 +54,10 @@ NOTICE: file4.txt: Skipped copy as --dry-run is set (size 0)
NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)
INFO : Resynching Path1 to Path2
NOTICE: file1.txt: Skipped copy as --dry-run is set (size 0)
NOTICE: file10.txt: Skipped delete as --dry-run is set (size 19)
NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)
NOTICE: file3.txt: Skipped copy as --dry-run is set (size 0)
NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)
NOTICE: file5.txt: Skipped copy (or update modification time) as --dry-run is set (size 39)
NOTICE: file6.txt: Skipped delete as --dry-run is set (size 19)
NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
INFO : Resync updating listings
INFO : Bisync successful

54
docs/content/bisync.md
View File

@ -91,6 +91,8 @@ Optional Flags:
If exceeded, the bisync run will abort. (default: 50%)
--force Bypass `--max-delete` safety check and run the sync.
Consider using with `--verbose`
--create-empty-src-dirs Sync creation and deletion of empty directories.
(Not compatible with --remove-empty-dirs)
--remove-empty-dirs Remove empty directories at the final cleanup step.
-1, --resync Performs the resync run.
Warning: Path1 files may overwrite Path2 versions.
@ -125,7 +127,7 @@ Cloud references are distinguished by having a `:` in the argument
(see [Windows support](#windows) below).
Path1 and Path2 are treated equally, in that neither has priority for
file changes, and access efficiency does not change whether a remote
file changes (except during [`--resync`](#resync)), and access efficiency does not change whether a remote
is on Path1 or Path2.
The listings in bisync working directory (default: `~/.cache/rclone/bisync`)
@ -134,8 +136,8 @@ to individual directories within the tree may be set up, e.g.:
`path_to_local_tree..dropbox_subdir.lst`.
Any empty directories after the sync on both the Path1 and Path2
filesystems are not deleted by default. If the `--remove-empty-dirs`
flag is specified, then both paths will have any empty directories purged
filesystems are not deleted by default, unless `--create-empty-src-dirs` is specified.
If the `--remove-empty-dirs` flag is specified, then both paths will have ALL empty directories purged
as the last step in the process.
## Command-line flags
@ -144,15 +146,31 @@ as the last step in the process.
This will effectively make both Path1 and Path2 filesystems contain a
matching superset of all files. Path2 files that do not exist in Path1 will
be copied to Path1, and the process will then sync the Path1 tree to Path2.
be copied to Path1, and the process will then copy the Path1 tree to Path2.
The base directories on the both Path1 and Path2 filesystems must exist
The `--resync` sequence is roughly equivalent to:
```
rclone copy Path2 Path1 --ignore-existing
rclone copy Path1 Path2
```
Or, if using `--create-empty-src-dirs`:
```
rclone copy Path2 Path1 --ignore-existing
rclone copy Path1 Path2 --create-empty-src-dirs
rclone copy Path2 Path1 --create-empty-src-dirs
```
The base directories on both Path1 and Path2 filesystems must exist
or bisync will fail. This is required for safety - that bisync can verify
that both paths are valid.
When using `--resync`, a newer version of a file either on Path1 or Path2
filesystem, will overwrite the file on the other path (only the last version
will be kept). Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags).
When using `--resync`, a newer version of a file on the Path2 filesystem
will be overwritten by the Path1 filesystem version.
(Note that this is [NOT entirely symmetrical](https://github.com/rclone/rclone/issues/5681#issuecomment-938761815).)
Carefully evaluate deltas using [--dry-run](/flags/#non-backend-flags).
[//]: # (I reverted a recent change in the above paragraph, as it was incorrect.
https://github.com/rclone/rclone/commit/dd72aff98a46c6e20848ac7ae5f7b19d45802493 )
For a resync run, one of the paths may be empty (no files in the path tree).
The resync run should result in files on both paths, else a normal non-resync
@ -493,6 +511,22 @@ rclone copy PATH1 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs
rclone copy PATH2 PATH2 --filter "+ */" --filter "- **" --create-empty-src-dirs
```
### Empty directories
By default, new/deleted empty directories on one path are _not_ propagated to the other side.
This is because bisync (and rclone) natively works on files, not directories.
However, this can be changed with the `--create-empty-src-dirs` flag, which works in
much the same way as in [`sync`](/commands/rclone_sync/) and [`copy`](/commands/rclone_copy/).
When used, empty directories created or deleted on one side will also be created or deleted on the other side.
The following should be noted:
* `--create-empty-src-dirs` is not compatible with `--remove-empty-dirs`. Use only one or the other (or neither).
* It is not recommended to switch back and forth between `--create-empty-src-dirs`
and the default (no `--create-empty-src-dirs`) without running `--resync`.
This is because it may appear as though all directories (not just the empty ones) were created/deleted,
when actually you've just toggled between making them visible/invisible to bisync.
It looks scarier than it is, but it's still probably best to stick to one or the other,
and use `--resync` when you need to switch.
### Renamed directories
Renaming a folder on the Path1 side results in deleting all files on
@ -1187,11 +1221,15 @@ about _Unison_ and synchronization in general.
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)
causing dry runs to inadvertently commit filter changes
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20%2D%2Dresync%20deletes%20data%2C%20contrary%20to%20docs)
causing `--resync` to erroneously delete empty folders and duplicate files unique to Path2
* `--check-access` is now enforced during `--resync`, preventing data loss in [certain user error scenarios](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=%2D%2Dcheck%2Daccess%20doesn%27t%20always%20fail%20when%20it%20should)
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=5.%20Bisync%20reads%20files%20in%20excluded%20directories%20during%20delete%20operations)
causing bisync to consider more files than necessary due to overbroad filters during delete operations
* [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides)
(identical files are now left alone instead of renamed)
* Added [support for `--create-empty-src-dirs`](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=3.%20Bisync%20should%20create/delete%20empty%20directories%20as%20sync%20does%2C%20when%20%2D%2Dcreate%2Dempty%2Dsrc%2Ddirs%20is%20passed)
* Added experimental `--resilient` mode to allow [recovery from self-correctable errors](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=2.%20Bisync%20should%20be%20more%20resilient%20to%20self%2Dcorrectable%20errors)
* Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes)
to distinguish from `--ignore-checksum`
* [Performance improvements](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20Deletes%20take%20several%20times%20longer%20than%20copies) for large remotes