From 8df78f2b6db0a4458ff2c2e6bebe536ccf5d2027 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 31 Jan 2018 16:15:30 +0000 Subject: [PATCH] operations: ignore size of objects when they are < 0 #320 This allows google docs to be transferred and checked correctly. --- docs/content/drive.md | 39 ++++++++++------------- fs/operations/operations.go | 21 ++++++++---- fs/operations/operations_internal_test.go | 37 +++++++++++++++++++++ 3 files changed, 67 insertions(+), 30 deletions(-) diff --git a/docs/content/drive.md b/docs/content/drive.md index 508bea8ed..60fad5312 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -382,13 +382,20 @@ see User rate limit exceeded errors, wait at least 24 hours and retry. You can disable server side copies with `--disable copy` to download and upload the files if you prefer. +#### Limitations of Google Docs #### + Google docs will appear as size -1 in `rclone ls` and as size 0 in -anything which uses the VFS layer, eg `rclone mount`, `rclone serve -XXX`. This is because rclone can't find out the size of the Google -Documents until they are downloaded. An unfortunate consequence of -this is that you can't download Google docs using `rclone mount` - you -will get a 0 sized file. If you try again the doc may gain its -correct size and be downloadable. +anything which uses the VFS layer, eg `rclone mount`, `rclone serve`. + +This is because rclone can't find out the size of the Google docs +without downloading them. + +Google docs will transfer correctly with `rclone sync`, `rclone copy` +etc as rclone knows to ignore the size when doing the transfer. + +However an unfortunate consequence of this is that you can't download +Google docs using `rclone mount` - you will get a 0 sized file. If +you try again the doc may gain its correct size and be downloadable. ### Duplicated files ### @@ -406,23 +413,9 @@ Android duplicates files on drive sometimes. ### Rclone appears to be re-copying files it shouldn't ### -There are two possible reasons for rclone to recopy files which -haven't changed to Google Drive. - -The first is the duplicated file issue above - run `rclone dedupe` and -check your logs for duplicate object or directory messages. - -The second is that sometimes Google reports different sizes for the -Google Docs exports which will cause rclone to re-download Google Docs -for no apparent reason. `--ignore-size` is a not very satisfactory -work-around for this if it is causing you a lot of problems. - -### Google docs downloads sometimes fail with "Failed to copy: read X bytes expecting Y" ### - -This is the same problem as above. Google reports the google doc is -one size, but rclone downloads a different size. Work-around with the -`--ignore-size` flag or wait for rclone to retry the download which it -will. +The most likely cause of this is the duplicated file issue above - run +`rclone dedupe` and check your logs for duplicate object or directory +messages. ### Making your own client_id ### diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a115fbbe6..4fb21c237 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -96,12 +96,19 @@ func Equal(src fs.ObjectInfo, dst fs.Object) bool { return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum) } +// sizeDiffers compare the size of src and dst taking into account the +// various ways of ignoring sizes +func sizeDiffers(src, dst fs.ObjectInfo) bool { + if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 { + return false + } + return src.Size() != dst.Size() +} + func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool { - if !fs.Config.IgnoreSize { - if src.Size() != dst.Size() { - fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size()) - return false - } + if sizeDiffers(src, dst) { + fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size()) + return false } if sizeOnly { fs.Debugf(src, "Sizes identical") @@ -310,7 +317,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec } // Verify sizes are the same after transfer - if !fs.Config.IgnoreSize && src.Size() != dst.Size() { + if sizeDiffers(src, dst) { err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size()) fs.Errorf(dst, "%v", err) fs.CountError(err) @@ -599,7 +606,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) { func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) { accounting.Stats.Checking(src.Remote()) defer accounting.Stats.DoneChecking(src.Remote()) - if !fs.Config.IgnoreSize && src.Size() != dst.Size() { + if sizeDiffers(src, dst) { err := errors.Errorf("Sizes differ") fs.Errorf(src, "%v", err) fs.CountError(err) diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go index 039a47788..11bb4f158 100644 --- a/fs/operations/operations_internal_test.go +++ b/fs/operations/operations_internal_test.go @@ -1,3 +1,40 @@ // Internal tests for operations package operations + +import ( + "fmt" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/object" + "github.com/stretchr/testify/assert" +) + +func TestSizeDiffers(t *testing.T) { + when := time.Now() + for _, test := range []struct { + ignoreSize bool + srcSize int64 + dstSize int64 + want bool + }{ + {false, 0, 0, false}, + {false, 1, 2, true}, + {false, 1, -1, false}, + {false, -1, 1, false}, + {true, 0, 0, false}, + {true, 1, 2, false}, + {true, 1, -1, false}, + {true, -1, 1, false}, + } { + src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil) + dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil) + oldIgnoreSize := fs.Config.IgnoreSize + fs.Config.IgnoreSize = test.ignoreSize + got := sizeDiffers(src, dst) + fs.Config.IgnoreSize = oldIgnoreSize + assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize)) + } +}