Add a delete threshold to sync (--max-delete)

Fixes #959

fs/accounting/stats.go

@@ -30,6 +30,7 @@ type StatsInfo struct {
 	checking     stringSet
 	transfers    int64
 	transferring stringSet
+	deletes      int64
 	start        time.Time
 	inProgress   *inProgress
 }
@@ -115,6 +116,14 @@ func (s *StatsInfo) GetLastError() error {
 	return s.lastError
 }
 
+// Deletes updates the stats for deletes
+func (s *StatsInfo) Deletes(deletes int64) int64 {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.deletes += deletes
+	return s.deletes
+}
+
 // ResetCounters sets the counters (bytes, checks, errors, transfers) to 0
 func (s *StatsInfo) ResetCounters() {
 	s.lock.RLock()
@@ -123,6 +132,7 @@ func (s *StatsInfo) ResetCounters() {
 	s.errors = 0
 	s.checks = 0
 	s.transfers = 0
+	s.deletes = 0
 }
 
 // ResetErrors sets the errors count to 0

fs/config.go

@@ -41,6 +41,7 @@ type ConfigInfo struct {
 	Dump                  DumpFlags
 	InsecureSkipVerify    bool // Skip server certificate verification
 	DeleteMode            DeleteMode
+	MaxDelete             int64
 	TrackRenames          bool // Track file renames.
 	LowLevelRetries       int
 	UpdateOlder           bool // Skip files that are newer on the destination
@@ -82,6 +83,7 @@ func NewConfig() *ConfigInfo {
 	c.ConnectTimeout = 60 * time.Second
 	c.Timeout = 5 * 60 * time.Second
 	c.DeleteMode = DeleteModeDefault
+	c.MaxDelete = -1
 	c.LowLevelRetries = 10
 	c.MaxDepth = -1
 	c.DataRateUnit = "bytes"

fs/config/configflags/configflags.go

@@ -53,6 +53,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transfering")
 	flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer (default)")
 	flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transfering")
+	flags.IntVar64P(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes")
 	flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible")
 	flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.")
 	flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.")

fs/config/flags/flags.go

@@ -89,6 +89,14 @@ func Int64P(name, shorthand string, value int64, usage string) (out *int64) {
 	return out
 }
 
+// IntVar64P defines a flag which can be overridden by an environment variable
+//
+// It is a thin wrapper around pflag.Int64VarP
+func IntVar64P(flags *pflag.FlagSet, p *int64, name, shorthand string, value int64, usage string) {
+	flags.Int64VarP(p, name, shorthand, value, usage)
+	setDefaultFromEnv(name)
+}
+
 // IntVarP defines a flag which can be overridden by an environment variable
 //
 // It is a thin wrapper around pflag.IntVarP

fs/operations/operations.go

@@ -416,6 +416,10 @@ func CanServerSideMove(fdst fs.Fs) bool {
 // deleting
 func DeleteFileWithBackupDir(dst fs.Object, backupDir fs.Fs) (err error) {
 	accounting.Stats.Checking(dst.Remote())
+	numDeletes := accounting.Stats.Deletes(1)
+	if fs.Config.MaxDelete != -1 && numDeletes > fs.Config.MaxDelete {
+		return fserrors.FatalError(errors.New("--max-delete threshold reached"))
+	}
 	action, actioned, actioning := "delete", "Deleted", "deleting"
 	if backupDir != nil {
 		action, actioned, actioning = "move into backup dir", "Moved into backup dir", "moving into backup dir"
@@ -460,6 +464,8 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error
 	var wg sync.WaitGroup
 	wg.Add(fs.Config.Transfers)
 	var errorCount int32
+	var fatalErrorCount int32
+
 	for i := 0; i < fs.Config.Transfers; i++ {
 		go func() {
 			defer wg.Done()
@@ -467,6 +473,11 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error
 				err := DeleteFileWithBackupDir(dst, backupDir)
 				if err != nil {
 					atomic.AddInt32(&errorCount, 1)
+					if fserrors.IsFatalError(err) {
+						fs.Errorf(nil, "Got fatal error on delete: %s", err)
+						atomic.AddInt32(&fatalErrorCount, 1)
+						return
+					}
 				}
 			}
 		}()
@@ -474,7 +485,11 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error
 	fs.Infof(nil, "Waiting for deletions to finish")
 	wg.Wait()
 	if errorCount > 0 {
-		return errors.Errorf("failed to delete %d files", errorCount)
+		err := errors.Errorf("failed to delete %d files", errorCount)
+		if fatalErrorCount > 0 {
+			return fserrors.FatalError(err)
+		}
+		return err
 	}
 	return nil
 }