From 8e8b78d7e57fae885a30df4d57eaa1da8f03dbd5 Mon Sep 17 00:00:00 2001 From: yparitcher Date: Sun, 7 Jul 2019 21:02:53 -0400 Subject: [PATCH] Implement --compare-dest & --copy-dest Fixes #3278 --- docs/content/docs.md | 27 ++++ fs/config.go | 2 + fs/config/configflags/configflags.go | 6 + fs/operations/operations.go | 145 ++++++++++++++++++-- fs/operations/operations_test.go | 177 +++++++++++++++++++++++++ fs/sync/sync.go | 101 ++++++++------ fs/sync/sync_test.go | 190 +++++++++++++++++++++++++++ 7 files changed, 601 insertions(+), 47 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 278c2b894..5d14b1647 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -328,6 +328,8 @@ If running rclone from a script you might want to use today's date as the directory name passed to `--backup-dir` to store the old files, or you might want to pass `--suffix` with today's date. +See `--compare-dest` and `--copy-dest`. + ### --bind string ### Local address to bind to for outgoing connections. This can be an @@ -447,6 +449,18 @@ quicker than without the `--checksum` flag. When using this flag, rclone won't update mtimes of remote files if they are incorrect as it would normally. +### --compare-dest=DIR ### + +When using `sync`, `copy` or `move` DIR is checked in addition to the +destination for files. If a file identical to the source is found that +file is NOT copied from source. This is useful to copy just files that +have changed since the last backup. + +You must use the same remote as the destination of the sync. The +compare directory must not overlap the destination directory. + +See `--copy-dest` and `--backup-dir`. + ### --config=CONFIG_FILE ### Specify the location of the rclone config file. @@ -475,6 +489,19 @@ The connection timeout is the amount of time rclone will wait for a connection to go through to a remote object storage system. It is `1m` by default. +### --copy-dest=DIR ### + +When using `sync`, `copy` or `move` DIR is checked in addition to the +destination for files. If a file identical to the source is found that +file is server side copied from DIR to the destination. This is useful +for incremental backup. + +The remote in use must support server side copy and you must +use the same remote as the destination of the sync. The compare +directory must not overlap the destination directory. + +See `--compare-dest` and `--backup-dir`. + ### --dedupe-mode MODE ### Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean. diff --git a/fs/config.go b/fs/config.go index 6b77fc700..6817df55a 100644 --- a/fs/config.go +++ b/fs/config.go @@ -66,6 +66,8 @@ type ConfigInfo struct { NoTraverse bool NoUpdateModTime bool DataRateUnit string + CompareDest string + CopyDest string BackupDir string Suffix string SuffixKeepExtension bool diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 97e5cbe11..99c9fead7 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -67,6 +67,8 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing") flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.") flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") + flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "use DIR to server side copy flies from.") + flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Compare dest to DIR also.") flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.") flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.") flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.") @@ -152,6 +154,10 @@ func SetFlags() { log.Fatalf(`Can't use --size-only and --ignore-size together.`) } + if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" { + log.Fatalf(`Can't use --compare-dest with --copy-dest.`) + } + switch { case len(fs.Config.StatsOneLineDateFormat) > 0: fs.Config.StatsOneLineDate = true diff --git a/fs/operations/operations.go b/fs/operations/operations.go index b629c715c..3529a847a 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -96,7 +96,7 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b // Otherwise the file is considered to be not equal including if there // were errors reading info. func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool { - return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum) + return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum, !fs.Config.NoUpdateModTime) } // sizeDiffers compare the size of src and dst taking into account the @@ -110,7 +110,7 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool { var checksumWarning sync.Once -func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool { +func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum, UpdateModTime bool) bool { if sizeDiffers(src, dst) { fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size()) return false @@ -169,7 +169,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, chec } // mod time differs but hash is the same to reset mod time if required - if !fs.Config.NoUpdateModTime { + if UpdateModTime { if fs.Config.DryRun { fs.Logf(src, "Not updating modification time as --dry-run") } else { @@ -1360,6 +1360,115 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error { return nil } +// GetCompareDest sets up --compare-dest +func GetCompareDest() (CompareDest fs.Fs, err error) { + CompareDest, err = cache.Get(fs.Config.CompareDest) + if err != nil { + return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err)) + } + return CompareDest, nil +} + +// compareDest checks --compare-dest to see if src needs to +// be copied +// +// Returns True if src is in --compare-dest +func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CompareDestFile, err := CompareDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + if Equal(ctx, src, CompareDestFile) { + fs.Debugf(src, "Destination found in --compare-dest, skipping") + return true, nil + } + return false, nil +} + +// GetCopyDest sets up --copy-dest +func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error) { + CopyDest, err = cache.Get(fs.Config.CopyDest) + if err != nil { + return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err)) + } + if !SameConfig(fdst, CopyDest) { + return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination")) + } + if CopyDest.Features().Copy == nil { + return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy")) + } + return CopyDest, nil +} + +// copyDest checks --copy-dest to see if src needs to +// be copied +// +// Returns True if src was copied from --copy-dest +func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CopyDestFile, err := CopyDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + if equal(ctx, src, CopyDestFile, fs.Config.SizeOnly, fs.Config.CheckSum, false) { + if dst == nil || !Equal(ctx, src, dst) { + if dst != nil && backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dst) + if err != nil { + return false, errors.Wrap(err, "moving to --backup-dir failed") + } + // If successful zero out the dstObj as it is no longer there + dst = nil + } + _, err := Copy(ctx, fdst, dst, remote, CopyDestFile) + if err != nil { + fs.Errorf(src, "Destination found in --copy-dest, error copying") + return false, nil + } + fs.Debugf(src, "Destination found in --copy-dest, using server side copy") + return true, nil + } + fs.Debugf(src, "Unchanged skipping") + return true, nil + } + fs.Debugf(src, "Destination not found in --copy-dest") + return false, nil +} + +// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src +// does not need to be copied +// +// Returns True if src does not need to be copied +func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + if fs.Config.CompareDest != "" { + return compareDest(ctx, dst, src, CompareOrCopyDest) + } else if fs.Config.CopyDest != "" { + return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir) + } + return false, nil +} + // NeedTransfer checks to see if src needs to be copied to dst using // the current config. // @@ -1577,13 +1686,31 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str return err } - if NeedTransfer(ctx, dstObj, srcObj) { + var backupDir, copyDestDir fs.Fs + if fs.Config.BackupDir != "" || fs.Config.Suffix != "" { + backupDir, err = BackupDir(fdst, fsrc, srcFileName) + if err != nil { + return errors.Wrap(err, "creating Fs for --backup-dir failed") + } + } + if fs.Config.CompareDest != "" { + copyDestDir, err = GetCompareDest() + if err != nil { + return err + } + } else if fs.Config.CopyDest != "" { + copyDestDir, err = GetCopyDest(fdst) + if err != nil { + return err + } + } + NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir) + if err != nil { + return err + } + if !NoNeedTransfer && NeedTransfer(ctx, dstObj, srcObj) { // If destination already exists, then we must move it into --backup-dir if required - if dstObj != nil && (fs.Config.BackupDir != "" || fs.Config.Suffix != "") { - backupDir, err := BackupDir(fdst, fsrc, srcFileName) - if err != nil { - return errors.Wrap(err, "creating Fs for --backup-dir failed") - } + if dstObj != nil && backupDir != nil { err = MoveBackupDir(ctx, backupDir, dstObj) if err != nil { return errors.Wrap(err, "moving to --backup-dir failed") diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 44dd9cbf9..75f84373a 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -866,6 +866,183 @@ func TestCopyFileBackupDir(t *testing.T) { fstest.CheckItems(t, r.Fremote, file1old, file1) } +// Test with CompareDest set +func TestCopyFileCompareDest(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.CompareDest = r.FremoteName + "/CompareDest" + defer func() { + fs.Config.CompareDest = "" + }() + fdst, err := fs.NewFs(r.FremoteName + "/dst") + require.NoError(t, err) + + // check empty dest, empty compare + file1 := r.WriteFile("one", "one", t1) + fstest.CheckItems(t, r.Flocal, file1) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1dst) + + // check old dest, empty compare + file1b := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file1dst) + fstest.CheckItems(t, r.Flocal, file1b) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1bdst) + + // check old dest, new compare + file3 := r.WriteObject(context.Background(), "dst/one", "one", t1) + file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file2, file3) + fstest.CheckItems(t, r.Flocal, file1c) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3) + + // check empty dest, new compare + file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + + // check new dest, new compare + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + + // check empty dest, old compare + file5b := r.WriteFile("two", "twot3", t3) + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5b) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5b.Path, file5b.Path) + require.NoError(t, err) + + file5bdst := file5b + file5bdst.Path = "dst/two" + + fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst) +} + +// Test with CopyDest set +func TestCopyFileCopyDest(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + if r.Fremote.Features().Copy == nil { + t.Skip("Skipping test as remote does not support server side copy") + } + + fs.Config.CopyDest = r.FremoteName + "/CopyDest" + defer func() { + fs.Config.CopyDest = "" + }() + + fdst, err := fs.NewFs(r.FremoteName + "/dst") + require.NoError(t, err) + + // check empty dest, empty copy + file1 := r.WriteFile("one", "one", t1) + fstest.CheckItems(t, r.Flocal, file1) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1dst) + + // check old dest, empty copy + file1b := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file1dst) + fstest.CheckItems(t, r.Flocal, file1b) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1bdst) + + // check old dest, new copy, backup-dir + + fs.Config.BackupDir = r.FremoteName + "/BackupDir" + + file3 := r.WriteObject(context.Background(), "dst/one", "one", t1) + file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file2, file3) + fstest.CheckItems(t, r.Flocal, file1c) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path) + require.NoError(t, err) + + file2dst := file2 + file2dst.Path = "dst/one" + file3.Path = "BackupDir/one" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3) + fs.Config.BackupDir = "" + + // check empty dest, new copy + file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + file4dst := file4 + file4dst.Path = "dst/two" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) + + // check new dest, new copy + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) + + // check empty dest, old copy + file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2) + file7 := r.WriteFile("three", "threet3", t3) + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6) + fstest.CheckItems(t, r.Flocal, file1c, file5, file7) + + err = operations.CopyFile(context.Background(), fdst, r.Flocal, file7.Path, file7.Path) + require.NoError(t, err) + + file7dst := file7 + file7dst.Path = "dst/three" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst) +} + // testFsInfo is for unit testing fs.Info type testFsInfo struct { name string diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 903998847..f96bdcd5b 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -28,39 +28,40 @@ type syncCopyMove struct { deleteEmptySrcDirs bool dir string // internal state - ctx context.Context // internal context for controlling go-routines - cancel func() // cancel the context - noTraverse bool // if set don't traverse the dst - deletersWg sync.WaitGroup // for delete before go routine - deleteFilesCh chan fs.Object // channel to receive deletes if delete before - trackRenames bool // set if we should do server side renames - dstFilesMu sync.Mutex // protect dstFiles - dstFiles map[string]fs.Object // dst files, always filled - srcFiles map[string]fs.Object // src files, only used if deleteBefore - srcFilesChan chan fs.Object // passes src objects - srcFilesResult chan error // error result of src listing - dstFilesResult chan error // error result of dst listing - dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs - dstEmptyDirs map[string]fs.DirEntry // potentially empty directories - srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs - srcEmptyDirs map[string]fs.DirEntry // potentially empty directories - checkerWg sync.WaitGroup // wait for checkers - toBeChecked *pipe // checkers channel - transfersWg sync.WaitGroup // wait for transfers - toBeUploaded *pipe // copiers channel - errorMu sync.Mutex // Mutex covering the errors variables - err error // normal error from copy process - noRetryErr error // error with NoRetry set - fatalErr error // fatal error - commonHash hash.Type // common hash type between src and dst - renameMapMu sync.Mutex // mutex to protect the below - renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames - renamerWg sync.WaitGroup // wait for renamers - toBeRenamed *pipe // renamers channel - trackRenamesWg sync.WaitGroup // wg for background track renames - trackRenamesCh chan fs.Object // objects are pumped in here - renameCheck []fs.Object // accumulate files to check for rename here - backupDir fs.Fs // place to store overwrites/deletes + ctx context.Context // internal context for controlling go-routines + cancel func() // cancel the context + noTraverse bool // if set don't traverse the dst + deletersWg sync.WaitGroup // for delete before go routine + deleteFilesCh chan fs.Object // channel to receive deletes if delete before + trackRenames bool // set if we should do server side renames + dstFilesMu sync.Mutex // protect dstFiles + dstFiles map[string]fs.Object // dst files, always filled + srcFiles map[string]fs.Object // src files, only used if deleteBefore + srcFilesChan chan fs.Object // passes src objects + srcFilesResult chan error // error result of src listing + dstFilesResult chan error // error result of dst listing + dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs + dstEmptyDirs map[string]fs.DirEntry // potentially empty directories + srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs + srcEmptyDirs map[string]fs.DirEntry // potentially empty directories + checkerWg sync.WaitGroup // wait for checkers + toBeChecked *pipe // checkers channel + transfersWg sync.WaitGroup // wait for transfers + toBeUploaded *pipe // copiers channel + errorMu sync.Mutex // Mutex covering the errors variables + err error // normal error from copy process + noRetryErr error // error with NoRetry set + fatalErr error // fatal error + commonHash hash.Type // common hash type between src and dst + renameMapMu sync.Mutex // mutex to protect the below + renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames + renamerWg sync.WaitGroup // wait for renamers + toBeRenamed *pipe // renamers channel + trackRenamesWg sync.WaitGroup // wg for background track renames + trackRenamesCh chan fs.Object // objects are pumped in here + renameCheck []fs.Object // accumulate files to check for rename here + compareCopyDest fs.Fs // place to check for files to server side copy + backupDir fs.Fs // place to store overwrites/deletes } func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) { @@ -127,6 +128,19 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete return nil, err } } + if fs.Config.CompareDest != "" { + var err error + s.compareCopyDest, err = operations.GetCompareDest() + if err != nil { + return nil, err + } + } else if fs.Config.CopyDest != "" { + var err error + s.compareCopyDest, err = operations.GetCopyDest(fdst) + if err != nil { + return nil, err + } + } return s, nil } @@ -204,7 +218,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) { accounting.Stats.Checking(src.Remote()) // Check to see if can store this if src.Storable() { - if operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) { + NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir) + if err != nil { + s.processError(err) + } + if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) { // If files are treated as immutable, fail if destination exists and does not match if fs.Config.Immutable && pair.Dst != nil { fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified") @@ -764,10 +782,17 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { case s.trackRenamesCh <- x: } } else { - // No need to check since doesn't exist - ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil}) - if !ok { - return + // Check CompareDest && CopyDest + NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir) + if err != nil { + s.processError(err) + } + if !NoNeedTransfer { + // No need to check since doesn't exist + ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil}) + if !ok { + return + } } } case fs.Directory: diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 6c9929b81..d721601ed 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1216,6 +1216,196 @@ func TestSyncOverlap(t *testing.T) { checkErr(Sync(context.Background(), FremoteSync, FremoteSync, false)) } +// Test with CompareDest set +func TestSyncCompareDest(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.CompareDest = r.FremoteName + "/CompareDest" + defer func() { + fs.Config.CompareDest = "" + }() + + fdst, err := fs.NewFs(r.FremoteName + "/dst") + require.NoError(t, err) + + // check empty dest, empty compare + file1 := r.WriteFile("one", "one", t1) + fstest.CheckItems(t, r.Flocal, file1) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1dst) + + // check old dest, empty compare + file1b := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file1dst) + fstest.CheckItems(t, r.Flocal, file1b) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1bdst) + + // check old dest, new compare + file3 := r.WriteObject(context.Background(), "dst/one", "one", t1) + file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file2, file3) + fstest.CheckItems(t, r.Flocal, file1c) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3) + + // check empty dest, new compare + file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + + // check new dest, new compare + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + + // check empty dest, old compare + file5b := r.WriteFile("two", "twot3", t3) + fstest.CheckItems(t, r.Fremote, file2, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5b) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file5bdst := file5b + file5bdst.Path = "dst/two" + + fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst) +} + +// Test with CopyDest set +func TestSyncCopyDest(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + if r.Fremote.Features().Copy == nil { + t.Skip("Skipping test as remote does not support server side copy") + } + + fs.Config.CopyDest = r.FremoteName + "/CopyDest" + defer func() { + fs.Config.CopyDest = "" + }() + + fdst, err := fs.NewFs(r.FremoteName + "/dst") + require.NoError(t, err) + + // check empty dest, empty copy + file1 := r.WriteFile("one", "one", t1) + fstest.CheckItems(t, r.Flocal, file1) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1dst) + + // check old dest, empty copy + file1b := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file1dst) + fstest.CheckItems(t, r.Flocal, file1b) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + fstest.CheckItems(t, r.Fremote, file1bdst) + + // check old dest, new copy, backup-dir + + fs.Config.BackupDir = r.FremoteName + "/BackupDir" + + file3 := r.WriteObject(context.Background(), "dst/one", "one", t1) + file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + fstest.CheckItems(t, r.Fremote, file2, file3) + fstest.CheckItems(t, r.Flocal, file1c) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file2dst := file2 + file2dst.Path = "dst/one" + file3.Path = "BackupDir/one" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3) + fs.Config.BackupDir = "" + + // check empty dest, new copy + file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4) + fstest.CheckItems(t, r.Flocal, file1c, file5) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file4dst := file4 + file4dst.Path = "dst/two" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) + + // check new dest, new copy + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst) + + // check empty dest, old copy + file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2) + file7 := r.WriteFile("three", "threet3", t3) + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6) + fstest.CheckItems(t, r.Flocal, file1c, file5, file7) + + accounting.Stats.ResetCounters() + err = Sync(context.Background(), fdst, r.Flocal, false) + require.NoError(t, err) + + file7dst := file7 + file7dst.Path = "dst/three" + + fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst) +} + // Test with BackupDir set func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) { r := fstest.NewRun(t)