github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-21 22:50:16 +01:00
Code Releases Activity

Implement --compare-dest & --copy-dest Fixes #3278

Browse Source
This commit is contained in:
yparitcher 2019-07-07 21:02:53 -04:00 committed by Nick Craig-Wood
parent 266600dba7
commit 8e8b78d7e5
7 changed files with 601 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
docs/content/docs.md
View File

@ -328,6 +328,8 @@ If running rclone from a script you might want to use today's date as
the directory name passed to `--backup-dir` to store the old files, or
you might want to pass `--suffix` with today's date.
See `--compare-dest` and `--copy-dest`.
### --bind string ###
Local address to bind to for outgoing connections. This can be an
@ -447,6 +449,18 @@ quicker than without the `--checksum` flag.
When using this flag, rclone won't update mtimes of remote files if
they are incorrect as it would normally.
### --compare-dest=DIR ###
When using `sync`, `copy` or `move` DIR is checked in addition to the
destination for files. If a file identical to the source is found that
file is NOT copied from source. This is useful to copy just files that
have changed since the last backup.
You must use the same remote as the destination of the sync. The
compare directory must not overlap the destination directory.
See `--copy-dest` and `--backup-dir`.
### --config=CONFIG_FILE ###
Specify the location of the rclone config file.
@ -475,6 +489,19 @@ The connection timeout is the amount of time rclone will wait for a
connection to go through to a remote object storage system. It is
`1m` by default.
### --copy-dest=DIR ###
When using `sync`, `copy` or `move` DIR is checked in addition to the
destination for files. If a file identical to the source is found that
file is server side copied from DIR to the destination. This is useful
for incremental backup.
The remote in use must support server side copy and you must
use the same remote as the destination of the sync. The compare
directory must not overlap the destination directory.
See `--compare-dest` and `--backup-dir`.
### --dedupe-mode MODE ###
Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean.

2
fs/config.go
View File

@ -66,6 +66,8 @@ type ConfigInfo struct {
NoTraverse bool
NoUpdateModTime bool
DataRateUnit string
CompareDest string
CopyDest string
BackupDir string
Suffix string
SuffixKeepExtension bool

6
fs/config/configflags/configflags.go
View File

@ -67,6 +67,8 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "use DIR to server side copy flies from.")
flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Compare dest to DIR also.")
flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.")
flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@ -152,6 +154,10 @@ func SetFlags() {
log.Fatalf(`Can't use --size-only and --ignore-size together.`)
}
if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}
switch {
case len(fs.Config.StatsOneLineDateFormat) > 0:
fs.Config.StatsOneLineDate = true

145
fs/operations/operations.go
View File

@ -96,7 +96,7 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum, !fs.Config.NoUpdateModTime)
}
// sizeDiffers compare the size of src and dst taking into account the
@ -110,7 +110,7 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool {
var checksumWarning sync.Once
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum, UpdateModTime bool) bool {
if sizeDiffers(src, dst) {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
return false
@ -169,7 +169,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, chec
}
// mod time differs but hash is the same to reset mod time if required
if !fs.Config.NoUpdateModTime {
if UpdateModTime {
if fs.Config.DryRun {
fs.Logf(src, "Not updating modification time as --dry-run")
} else {
@ -1360,6 +1360,115 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
return nil
}
// GetCompareDest sets up --compare-dest
func GetCompareDest() (CompareDest fs.Fs, err error) {
CompareDest, err = cache.Get(fs.Config.CompareDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err))
}
return CompareDest, nil
}
// compareDest checks --compare-dest to see if src needs to
// be copied
//
// Returns True if src is in --compare-dest
func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) {
var remote string
if dst == nil {
remote = src.Remote()
} else {
remote = dst.Remote()
}
CompareDestFile, err := CompareDest.NewObject(ctx, remote)
switch err {
case fs.ErrorObjectNotFound:
return false, nil
case nil:
break
default:
return false, err
}
if Equal(ctx, src, CompareDestFile) {
fs.Debugf(src, "Destination found in --compare-dest, skipping")
return true, nil
}
return false, nil
}
// GetCopyDest sets up --copy-dest
func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error) {
CopyDest, err = cache.Get(fs.Config.CopyDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err))
}
if !SameConfig(fdst, CopyDest) {
return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
}
if CopyDest.Features().Copy == nil {
return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
}
return CopyDest, nil
}
// copyDest checks --copy-dest to see if src needs to
// be copied
//
// Returns True if src was copied from --copy-dest
func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
var remote string
if dst == nil {
remote = src.Remote()
} else {
remote = dst.Remote()
}
CopyDestFile, err := CopyDest.NewObject(ctx, remote)
switch err {
case fs.ErrorObjectNotFound:
return false, nil
case nil:
break
default:
return false, err
}
if equal(ctx, src, CopyDestFile, fs.Config.SizeOnly, fs.Config.CheckSum, false) {
if dst == nil || !Equal(ctx, src, dst) {
if dst != nil && backupDir != nil {
err = MoveBackupDir(ctx, backupDir, dst)
if err != nil {
return false, errors.Wrap(err, "moving to --backup-dir failed")
}
// If successful zero out the dstObj as it is no longer there
dst = nil
}
_, err := Copy(ctx, fdst, dst, remote, CopyDestFile)
if err != nil {
fs.Errorf(src, "Destination found in --copy-dest, error copying")
return false, nil
}
fs.Debugf(src, "Destination found in --copy-dest, using server side copy")
return true, nil
}
fs.Debugf(src, "Unchanged skipping")
return true, nil
}
fs.Debugf(src, "Destination not found in --copy-dest")
return false, nil
}
// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src
// does not need to be copied
//
// Returns True if src does not need to be copied
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
if fs.Config.CompareDest != "" {
return compareDest(ctx, dst, src, CompareOrCopyDest)
} else if fs.Config.CopyDest != "" {
return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
}
return false, nil
}
// NeedTransfer checks to see if src needs to be copied to dst using
// the current config.
//
@ -1577,13 +1686,31 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
return err
}
if NeedTransfer(ctx, dstObj, srcObj) {
var backupDir, copyDestDir fs.Fs
if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
backupDir, err = BackupDir(fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
}
if fs.Config.CompareDest != "" {
copyDestDir, err = GetCompareDest()
if err != nil {
return err
}
} else if fs.Config.CopyDest != "" {
copyDestDir, err = GetCopyDest(fdst)
if err != nil {
return err
}
}
NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir)
if err != nil {
return err
}
if !NoNeedTransfer && NeedTransfer(ctx, dstObj, srcObj) {
// If destination already exists, then we must move it into --backup-dir if required
if dstObj != nil && (fs.Config.BackupDir != "" || fs.Config.Suffix != "") {
backupDir, err := BackupDir(fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
if dstObj != nil && backupDir != nil {
err = MoveBackupDir(ctx, backupDir, dstObj)
if err != nil {
return errors.Wrap(err, "moving to --backup-dir failed")

177
fs/operations/operations_test.go
View File

@ -866,6 +866,183 @@ func TestCopyFileBackupDir(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1old, file1)
}
// Test with CompareDest set
func TestCopyFileCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
fs.Config.CompareDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty compare
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty compare
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new compare
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3)
// check empty dest, new compare
file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check new dest, new compare
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check empty dest, old compare
file5b := r.WriteFile("two", "twot3", t3)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5b.Path, file5b.Path)
require.NoError(t, err)
file5bdst := file5b
file5bdst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
}
// Test with CopyDest set
func TestCopyFileCopyDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if r.Fremote.Features().Copy == nil {
t.Skip("Skipping test as remote does not support server side copy")
}
fs.Config.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
fs.Config.CopyDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty copy
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty copy
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new copy, backup-dir
fs.Config.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
file2dst := file2
file2dst.Path = "dst/one"
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
fs.Config.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
file4dst := file4
file4dst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check new dest, new copy
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check empty dest, old copy
file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
file7 := r.WriteFile("three", "threet3", t3)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file7.Path, file7.Path)
require.NoError(t, err)
file7dst := file7
file7dst.Path = "dst/three"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}
// testFsInfo is for unit testing fs.Info
type testFsInfo struct {
name string

101
fs/sync/sync.go
View File

@ -28,39 +28,40 @@ type syncCopyMove struct {
deleteEmptySrcDirs bool
dir string
// internal state
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
noTraverse bool // if set don't traverse the dst
deletersWg sync.WaitGroup // for delete before go routine
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
trackRenames bool // set if we should do server side renames
dstFilesMu sync.Mutex // protect dstFiles
dstFiles map[string]fs.Object // dst files, always filled
srcFiles map[string]fs.Object // src files, only used if deleteBefore
srcFilesChan chan fs.Object // passes src objects
srcFilesResult chan error // error result of src listing
dstFilesResult chan error // error result of dst listing
dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs
dstEmptyDirs map[string]fs.DirEntry // potentially empty directories
srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs
srcEmptyDirs map[string]fs.DirEntry // potentially empty directories
checkerWg sync.WaitGroup // wait for checkers
toBeChecked *pipe // checkers channel
transfersWg sync.WaitGroup // wait for transfers
toBeUploaded *pipe // copiers channel
errorMu sync.Mutex // Mutex covering the errors variables
err error // normal error from copy process
noRetryErr error // error with NoRetry set
fatalErr error // fatal error
commonHash hash.Type // common hash type between src and dst
renameMapMu sync.Mutex // mutex to protect the below
renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames
renamerWg sync.WaitGroup // wait for renamers
toBeRenamed *pipe // renamers channel
trackRenamesWg sync.WaitGroup // wg for background track renames
trackRenamesCh chan fs.Object // objects are pumped in here
renameCheck []fs.Object // accumulate files to check for rename here
backupDir fs.Fs // place to store overwrites/deletes
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
noTraverse bool // if set don't traverse the dst
deletersWg sync.WaitGroup // for delete before go routine
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
trackRenames bool // set if we should do server side renames
dstFilesMu sync.Mutex // protect dstFiles
dstFiles map[string]fs.Object // dst files, always filled
srcFiles map[string]fs.Object // src files, only used if deleteBefore
srcFilesChan chan fs.Object // passes src objects
srcFilesResult chan error // error result of src listing
dstFilesResult chan error // error result of dst listing
dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs
dstEmptyDirs map[string]fs.DirEntry // potentially empty directories
srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs
srcEmptyDirs map[string]fs.DirEntry // potentially empty directories
checkerWg sync.WaitGroup // wait for checkers
toBeChecked *pipe // checkers channel
transfersWg sync.WaitGroup // wait for transfers
toBeUploaded *pipe // copiers channel
errorMu sync.Mutex // Mutex covering the errors variables
err error // normal error from copy process
noRetryErr error // error with NoRetry set
fatalErr error // fatal error
commonHash hash.Type // common hash type between src and dst
renameMapMu sync.Mutex // mutex to protect the below
renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames
renamerWg sync.WaitGroup // wait for renamers
toBeRenamed *pipe // renamers channel
trackRenamesWg sync.WaitGroup // wg for background track renames
trackRenamesCh chan fs.Object // objects are pumped in here
renameCheck []fs.Object // accumulate files to check for rename here
compareCopyDest fs.Fs // place to check for files to server side copy
backupDir fs.Fs // place to store overwrites/deletes
}
func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
@ -127,6 +128,19 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
return nil, err
}
}
if fs.Config.CompareDest != "" {
var err error
s.compareCopyDest, err = operations.GetCompareDest()
if err != nil {
return nil, err
}
} else if fs.Config.CopyDest != "" {
var err error
s.compareCopyDest, err = operations.GetCopyDest(fdst)
if err != nil {
return nil, err
}
}
return s, nil
}
@ -204,7 +218,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
accounting.Stats.Checking(src.Remote())
// Check to see if can store this
if src.Storable() {
if operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)
if err != nil {
s.processError(err)
}
if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
// If files are treated as immutable, fail if destination exists and does not match
if fs.Config.Immutable && pair.Dst != nil {
fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
@ -764,10 +782,17 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
case s.trackRenamesCh <- x:
}
} else {
// No need to check since doesn't exist
ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
// Check CompareDest && CopyDest
NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir)
if err != nil {
s.processError(err)
}
if !NoNeedTransfer {
// No need to check since doesn't exist
ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
}
}
}
case fs.Directory:

190
fs/sync/sync_test.go
View File

@ -1216,6 +1216,196 @@ func TestSyncOverlap(t *testing.T) {
checkErr(Sync(context.Background(), FremoteSync, FremoteSync, false))
}
// Test with CompareDest set
func TestSyncCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
fs.Config.CompareDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty compare
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty compare
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new compare
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3)
// check empty dest, new compare
file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check new dest, new compare
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check empty dest, old compare
file5b := r.WriteFile("two", "twot3", t3)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file5bdst := file5b
file5bdst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
}
// Test with CopyDest set
func TestSyncCopyDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if r.Fremote.Features().Copy == nil {
t.Skip("Skipping test as remote does not support server side copy")
}
fs.Config.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
fs.Config.CopyDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty copy
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty copy
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new copy, backup-dir
fs.Config.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file2dst := file2
file2dst.Path = "dst/one"
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
fs.Config.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file4dst := file4
file4dst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check new dest, new copy
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check empty dest, old copy
file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
file7 := r.WriteFile("three", "threet3", t3)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file7dst := file7
file7dst.Path = "dst/three"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}
// Test with BackupDir set
func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
r := fstest.NewRun(t)