From 4013bc4a4cd768e219d5a1e281daf8126d549474 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 11 Mar 2021 14:44:01 +0000 Subject: [PATCH] Fix excessive retries missing --max-duration timeout - fixes #4504 This change checks the context whenever rclone might retry, and doesn't retry if the current context has an error. This fixes the pathological behaviour of `--max-duration` refusing to exit because all the context deadline exceeded errors were being retried. This unfortunately meant changing the shouldRetry logic in every backend and doing a lot of context propagation. See: https://forum.rclone.org/t/add-flag-to-exit-immediately-when-max-duration-reached/22723 --- backend/amazonclouddrive/amazonclouddrive.go | 85 +++++------ backend/azureblob/azureblob.go | 33 +++-- backend/b2/b2.go | 9 +- backend/box/box.go | 31 ++-- backend/box/upload.go | 10 +- backend/dropbox/dropbox.go | 135 +++++++++--------- backend/fichier/api.go | 31 ++-- backend/fichier/object.go | 2 +- backend/filefabric/filefabric.go | 9 +- .../googlecloudstorage/googlecloudstorage.go | 29 ++-- backend/googlephotos/googlephotos.go | 31 ++-- backend/jottacloud/jottacloud.go | 25 ++-- backend/opendrive/opendrive.go | 41 +++--- backend/pcloud/pcloud.go | 35 ++--- backend/premiumizeme/premiumizeme.go | 25 ++-- backend/putio/error.go | 6 +- backend/putio/fs.go | 32 ++--- backend/putio/object.go | 10 +- backend/seafile/seafile.go | 5 +- backend/seafile/webapi.go | 52 +++---- backend/sharefile/sharefile.go | 25 ++-- backend/sharefile/upload.go | 2 +- backend/sugarsync/sugarsync.go | 33 +++-- backend/webdav/webdav.go | 27 ++-- backend/yandex/yandex.go | 34 +++-- backend/zoho/zoho.go | 23 +-- fs/fserrors/error.go | 17 +++ fs/fserrors/error_test.go | 19 +++ fs/operations/check.go | 2 +- fs/operations/operations.go | 8 +- fs/operations/operations_test.go | 8 +- 31 files changed, 474 insertions(+), 360 deletions(-) diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go index 49969654d..d93f2acf7 100644 --- a/backend/amazonclouddrive/amazonclouddrive.go +++ b/backend/amazonclouddrive/amazonclouddrive.go @@ -205,7 +205,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } if resp != nil { if resp.StatusCode == 401 { f.tokenRenewer.Invalidate() @@ -280,7 +283,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Renew the token in the background f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { - _, err := f.getRootInfo() + _, err := f.getRootInfo(ctx) return err }) @@ -288,14 +291,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e var resp *http.Response err = f.pacer.Call(func() (bool, error) { _, resp, err = f.c.Account.GetEndpoints() - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to get endpoints") } // Get rootID - rootInfo, err := f.getRootInfo() + rootInfo, err := f.getRootInfo(ctx) if err != nil || rootInfo.Id == nil { return nil, errors.Wrap(err, "failed to get root") } @@ -337,11 +340,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } // getRootInfo gets the root folder info -func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) { +func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { rootInfo, resp, err = f.c.Nodes.GetRoot() - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) return rootInfo, err } @@ -380,7 +383,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin var subFolder *acd.Folder err = f.pacer.Call(func() (bool, error) { subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf)) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if err == acd.ErrorNodeNotFound { @@ -407,7 +410,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, var info *acd.Folder err = f.pacer.Call(func() (bool, error) { info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf)) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -428,7 +431,7 @@ type listAllFn func(*acd.Node) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { query := "parents:" + dirID if directoriesOnly { query += " AND kind:" + folderKind @@ -449,7 +452,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly var resp *http.Response err = f.pacer.CallNoRetry(func() (bool, error) { nodes, resp, err = f.c.Nodes.GetNodes(&opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return false, err @@ -508,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e var iErr error for tries := 1; tries <= maxTries; tries++ { entries = nil - _, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool { + _, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool { remote := path.Join(dir, *node.Name) switch *node.Kind { case folderKind: @@ -667,7 +670,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . if ok { return false, nil } - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -708,7 +711,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, if err != nil { return nil, err } - err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false) + err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false) if err != nil { return nil, err } @@ -803,7 +806,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string var jsonStr string err = srcFs.pacer.Call(func() (bool, error) { jsonStr, err = srcInfo.GetMetadata() - return srcFs.shouldRetry(nil, err) + return srcFs.shouldRetry(ctx, nil, err) }) if err != nil { fs.Debugf(src, "DirMove error: error reading src metadata: %v", err) @@ -815,7 +818,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return err } - err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true) + err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true) if err != nil { return err } @@ -840,7 +843,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { if check { // check directory is empty empty := true - _, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool { + _, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool { switch *node.Kind { case folderKind: empty = false @@ -865,7 +868,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = node.Trash() - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -987,7 +990,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { var info *acd.File err = o.fs.pacer.Call(func() (bool, error) { info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf)) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { if err == acd.ErrorNodeNotFound { @@ -1044,7 +1047,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } else { in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers) } - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) return in, err } @@ -1067,7 +1070,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if ok { return false, nil } - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -1077,70 +1080,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // Remove a node -func (f *Fs) removeNode(info *acd.Node) error { +func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error { var resp *http.Response var err error err = f.pacer.Call(func() (bool, error) { resp, err = info.Trash() - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) return err } // Remove an object func (o *Object) Remove(ctx context.Context) error { - return o.fs.removeNode(o.info) + return o.fs.removeNode(ctx, o.info) } // Restore a node -func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) { +func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { newInfo, resp, err = info.Restore() - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) return newInfo, err } // Changes name of given node -func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) { +func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName)) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) return newInfo, err } // Replaces one parent with another, effectively moving the file. Leaves other // parents untouched. ReplaceParent cannot be used when the file is trashed. -func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error { +func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error { return f.pacer.Call(func() (bool, error) { resp, err := info.ReplaceParent(oldParentID, newParentID) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) } // Adds one additional parent to object. -func (f *Fs) addParent(info *acd.Node, newParentID string) error { +func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error { return f.pacer.Call(func() (bool, error) { resp, err := info.AddParent(newParentID) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) } // Remove given parent from object, leaving the other possible // parents untouched. Object can end up having no parents. -func (f *Fs) removeParent(info *acd.Node, parentID string) error { +func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error { return f.pacer.Call(func() (bool, error) { resp, err := info.RemoveParent(parentID) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) } // moveNode moves the node given from the srcLeaf,srcDirectoryID to // the dstLeaf,dstDirectoryID -func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) { +func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) { // fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID) cantMove := fs.ErrorCantMove if useDirErrorMsgs { @@ -1154,7 +1157,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s if srcLeaf != dstLeaf { // fs.Debugf(name, "renaming") - _, err = f.renameNode(srcInfo, dstLeaf) + _, err = f.renameNode(ctx, srcInfo, dstLeaf) if err != nil { fs.Debugf(name, "Move: quick path rename failed: %v", err) goto OnConflict @@ -1162,7 +1165,7 @@ func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, s } if srcDirectoryID != dstDirectoryID { // fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID) - err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID) + err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID) if err != nil { fs.Debugf(name, "Move: quick path parent replace failed: %v", err) return err @@ -1175,13 +1178,13 @@ OnConflict: fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.") // fs.Debugf(name, "Trashing file") - err = f.removeNode(srcInfo) + err = f.removeNode(ctx, srcInfo) if err != nil { fs.Debugf(name, "Move: remove node failed: %v", err) return err } // fs.Debugf(name, "Renaming file") - _, err = f.renameNode(srcInfo, dstLeaf) + _, err = f.renameNode(ctx, srcInfo, dstLeaf) if err != nil { fs.Debugf(name, "Move: rename node failed: %v", err) return err @@ -1189,19 +1192,19 @@ OnConflict: // note: replacing parent is forbidden by API, modifying them individually is // okay though // fs.Debugf(name, "Adding target parent") - err = f.addParent(srcInfo, dstDirectoryID) + err = f.addParent(ctx, srcInfo, dstDirectoryID) if err != nil { fs.Debugf(name, "Move: addParent failed: %v", err) return err } // fs.Debugf(name, "removing original parent") - err = f.removeParent(srcInfo, srcDirectoryID) + err = f.removeParent(ctx, srcInfo, srcDirectoryID) if err != nil { fs.Debugf(name, "Move: removeParent failed: %v", err) return err } // fs.Debugf(name, "Restoring") - _, err = f.restoreNode(srcInfo) + _, err = f.restoreNode(ctx, srcInfo) if err != nil { fs.Debugf(name, "Move: restoreNode node failed: %v", err) return err diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index e0478c207..3701de1a0 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -347,7 +347,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } // FIXME interpret special errors - more to do here if storageErr, ok := err.(azblob.StorageError); ok { switch storageErr.ServiceCode() { @@ -578,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Retry as specified by the documentation: // https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance token, err = GetMSIToken(ctx, userMSI) - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) if err != nil { @@ -594,7 +597,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e var refreshedToken adal.Token err := f.imdsPacer.Call(func() (bool, error) { refreshedToken, err = GetMSIToken(ctx, userMSI) - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) if err != nil { // Failed to refresh. @@ -803,7 +806,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC err := f.pacer.Call(func() (bool, error) { var err error response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options) - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) if err != nil { @@ -1029,7 +1032,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error { err := f.pacer.Call(func() (bool, error) { var err error response, err = f.svcURL.ListContainersSegment(ctx, marker, params) - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) if err != nil { return err @@ -1098,7 +1101,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error { } } } - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) }, nil) } @@ -1136,10 +1139,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error { return false, fs.ErrorDirNotFound } - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) } - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) }) } @@ -1212,7 +1215,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, err = f.pacer.Call(func() (bool, error) { startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil) - return f.shouldRetry(err) + return f.shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -1373,7 +1376,7 @@ func (o *Object) readMetaData() (err error) { var blobProperties *azblob.BlobGetPropertiesResponse err = o.fs.pacer.Call(func() (bool, error) { blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{}) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) if err != nil { // On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well @@ -1408,7 +1411,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { blob := o.getBlobReference() err := o.fs.pacer.Call(func() (bool, error) { _, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) if err != nil { return err @@ -1451,7 +1454,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var downloadResponse *azblob.DownloadResponse err = o.fs.pacer.Call(func() (bool, error) { downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{}) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "failed to open for download") @@ -1592,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Stream contents of the reader object to the given blob URL blockBlobURL := blob.ToBlockBlobURL() _, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) if err != nil { return err @@ -1620,7 +1623,7 @@ func (o *Object) Remove(ctx context.Context) error { ac := azblob.BlobAccessConditions{} return o.fs.pacer.Call(func() (bool, error) { _, err := blob.Delete(ctx, snapShotOptions, ac) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) } @@ -1649,7 +1652,7 @@ func (o *Object) SetTier(tier string) error { ctx := context.Background() err := o.fs.pacer.Call(func() (bool, error) { _, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}) - return o.fs.shouldRetry(err) + return o.fs.shouldRetry(ctx, err) }) if err != nil { diff --git a/backend/b2/b2.go b/backend/b2/b2.go index a6f45bf13..b5690a99e 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -305,7 +305,10 @@ var retryErrorCodes = []int{ // shouldRetryNoAuth returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } // For 429 or 503 errors look at the Retry-After: header and // set the retry appropriately, starting with a minimum of 1 // second if it isn't set. @@ -336,7 +339,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b } return true, err } - return f.shouldRetryNoReauth(resp, err) + return f.shouldRetryNoReauth(ctx, resp, err) } // errorHandler parses a non 2xx error response into an error @@ -504,7 +507,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error { } err := f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info) - return f.shouldRetryNoReauth(resp, err) + return f.shouldRetryNoReauth(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to authenticate") diff --git a/backend/box/box.go b/backend/box/box.go index 3cbcd9a51..7a222b162 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -317,7 +317,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } authRetry := false if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { @@ -548,7 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -585,7 +588,7 @@ OUTER: var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -740,7 +743,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error { } return f.pacer.Call(func() (bool, error) { resp, err := f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } @@ -767,7 +770,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "rmdir failed") @@ -839,7 +842,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var info *api.Item err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, ©File, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -877,7 +880,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) ( var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -895,7 +898,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &user) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to read user info") @@ -1008,7 +1011,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return info.SharedLink.URL, err } @@ -1026,7 +1029,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error { } return f.pacer.Call(func() (bool, error) { resp, err := f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } @@ -1048,7 +1051,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't list trash") @@ -1182,7 +1185,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, var info *api.Item err := o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return info, err } @@ -1215,7 +1218,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1255,7 +1258,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str } err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return err diff --git a/backend/box/upload.go b/backend/box/upload.go index 649198a52..a7ad6cc09 100644 --- a/backend/box/upload.go +++ b/backend/box/upload.go @@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return } @@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total err = o.fs.pacer.Call(func() (bool, error) { opts.Body = wrap(bytes.NewReader(chunk)) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -109,10 +109,10 @@ outer: err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) if err != nil { - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) } body, err = rest.ReadBody(resp) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) delay := defaultDelay var why string @@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return err } diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 406cb0677..d788b50da 100755 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -292,7 +292,10 @@ func (f *Fs) Features() *fs.Features { // shouldRetry returns a boolean as to whether this err deserves to be // retried. It returns the err as a convenience -func shouldRetry(err error) (bool, error) { +func shouldRetry(ctx context.Context, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } if err == nil { return false, err } @@ -425,7 +428,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if f.root == "" { return f, nil } - _, err := f.findSharedFile(f.root) + _, err := f.findSharedFile(ctx, f.root) f.root = "" if err == nil { return f, fs.ErrorIsFile @@ -445,7 +448,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } // root is not empty so we have find the right shared folder if it exists - id, err := f.findSharedFolder(dir) + id, err := f.findSharedFolder(ctx, dir) if err != nil { // if we didn't find the specified shared folder we have to bail out here return nil, err @@ -453,7 +456,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // we found the specified shared folder so let's mount it // this will add it to the users normal root namespace and allows us // to actually perform operations on it using the normal api endpoints. - err = f.mountSharedFolder(id) + err = f.mountSharedFolder(ctx, id) if err != nil { switch e := err.(type) { case sharing.MountFolderAPIError: @@ -477,7 +480,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e var acc *users.FullAccount err = f.pacer.Call(func() (bool, error) { acc, err = f.users.GetCurrentAccount() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "get current account failed") @@ -495,7 +498,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e f.setRoot(root) // See if the root is actually an object - _, err = f.getFileMetadata(f.slashRoot) + _, err = f.getFileMetadata(ctx, f.slashRoot) if err == nil { newRoot := path.Dir(f.root) if newRoot == "." { @@ -529,12 +532,12 @@ func (f *Fs) setRoot(root string) { } // getMetadata gets the metadata for a file or directory -func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) { +func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) { err = f.pacer.Call(func() (bool, error) { entry, err = f.srv.GetMetadata(&files.GetMetadataArg{ Path: f.opt.Enc.FromStandardPath(objPath), }) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { switch e := err.(type) { @@ -549,8 +552,8 @@ func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, } // getFileMetadata gets the metadata for a file -func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) { - entry, notFound, err := f.getMetadata(filePath) +func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) { + entry, notFound, err := f.getMetadata(ctx, filePath) if err != nil { return nil, err } @@ -565,8 +568,8 @@ func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err } // getDirMetadata gets the metadata for a directory -func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) { - entry, notFound, err := f.getMetadata(dirPath) +func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) { + entry, notFound, err := f.getMetadata(ctx, dirPath) if err != nil { return nil, err } @@ -583,7 +586,7 @@ func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err // Return an Object from a path // // If it can't be found it returns the error fs.ErrorObjectNotFound. -func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) { +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) { o := &Object{ fs: f, remote: remote, @@ -592,7 +595,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje if info != nil { err = o.setMetadataFromEntry(info) } else { - err = o.readEntryAndSetMetadata() + err = o.readEntryAndSetMetadata(ctx) } if err != nil { return nil, err @@ -604,14 +607,14 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { if f.opt.SharedFiles { - return f.findSharedFile(remote) + return f.findSharedFile(ctx, remote) } - return f.newObjectWithInfo(remote, nil) + return f.newObjectWithInfo(ctx, remote, nil) } // listSharedFoldersApi lists all available shared folders mounted and not mounted // we'll need the id later so we have to return them in original format -func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { +func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) { started := false var res *sharing.ListFoldersResult for { @@ -621,7 +624,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { } err := f.pacer.Call(func() (bool, error) { res, err = f.sharing.ListFolders(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -633,7 +636,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { } err := f.pacer.Call(func() (bool, error) { res, err = f.sharing.ListFoldersContinue(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "list continue") @@ -658,8 +661,8 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) { // findSharedFolder find the id for a given shared folder name // somewhat annoyingly there is no endpoint to query a shared folder by it's name // so our only option is to iterate over all shared folders -func (f *Fs) findSharedFolder(name string) (id string, err error) { - entries, err := f.listSharedFolders() +func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) { + entries, err := f.listSharedFolders(ctx) if err != nil { return "", err } @@ -672,20 +675,20 @@ func (f *Fs) findSharedFolder(name string) (id string, err error) { } // mountSharedFolder mount a shared folder to the root namespace -func (f *Fs) mountSharedFolder(id string) error { +func (f *Fs) mountSharedFolder(ctx context.Context, id string) error { arg := sharing.MountFolderArg{ SharedFolderId: id, } err := f.pacer.Call(func() (bool, error) { _, err := f.sharing.MountFolder(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return err } // listReceivedFiles lists shared the user as access to (note this means individual // files not files contained in shared folders) -func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { +func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) { started := false var res *sharing.ListFilesResult for { @@ -695,7 +698,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { } err := f.pacer.Call(func() (bool, error) { res, err = f.sharing.ListReceivedFiles(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -707,7 +710,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { } err := f.pacer.Call(func() (bool, error) { res, err = f.sharing.ListReceivedFilesContinue(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "list continue") @@ -734,8 +737,8 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) { return entries, nil } -func (f *Fs) findSharedFile(name string) (o *Object, err error) { - files, err := f.listReceivedFiles() +func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) { + files, err := f.listReceivedFiles(ctx) if err != nil { return nil, err } @@ -758,10 +761,10 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) { // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { if f.opt.SharedFiles { - return f.listReceivedFiles() + return f.listReceivedFiles(ctx) } if f.opt.SharedFolders { - return f.listSharedFolders() + return f.listSharedFolders(ctx) } root := f.slashRoot @@ -782,7 +785,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } err = f.pacer.Call(func() (bool, error) { res, err = f.srv.ListFolder(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { switch e := err.(type) { @@ -800,7 +803,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } err = f.pacer.Call(func() (bool, error) { res, err = f.srv.ListFolderContinue(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "list continue") @@ -830,7 +833,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id) entries = append(entries, d) } else if fileInfo != nil { - o, err := f.newObjectWithInfo(remote, fileInfo) + o, err := f.newObjectWithInfo(ctx, remote, fileInfo) if err != nil { return nil, err } @@ -879,7 +882,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { } // check directory doesn't exist - _, err := f.getDirMetadata(root) + _, err := f.getDirMetadata(ctx, root) if err == nil { return nil // directory exists already } else if err != fs.ErrorDirNotFound { @@ -896,7 +899,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { } err = f.pacer.Call(func() (bool, error) { _, err = f.srv.CreateFolderV2(&arg2) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return err } @@ -913,7 +916,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) if check { // check directory exists - _, err = f.getDirMetadata(root) + _, err = f.getDirMetadata(ctx, root) if err != nil { return errors.Wrap(err, "Rmdir") } @@ -930,7 +933,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) var res *files.ListFolderResult err = f.pacer.Call(func() (bool, error) { res, err = f.srv.ListFolder(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return errors.Wrap(err, "Rmdir") @@ -943,7 +946,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) // remove it err = f.pacer.Call(func() (bool, error) { _, err = f.srv.DeleteV2(&files.DeleteArg{Path: root}) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return err } @@ -996,7 +999,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var result *files.RelocationResult err = f.pacer.Call(func() (bool, error) { result, err = f.srv.CopyV2(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "copy failed") @@ -1057,7 +1060,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, var result *files.RelocationResult err = f.pacer.Call(func() (bool, error) { result, err = f.srv.MoveV2(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "move failed") @@ -1091,7 +1094,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var linkRes sharing.IsSharedLinkMetadata err = f.pacer.Call(func() (bool, error) { linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil && strings.Contains(err.Error(), @@ -1104,7 +1107,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var listRes *sharing.ListSharedLinksResult err = f.pacer.Call(func() (bool, error) { listRes, err = f.sharing.ListSharedLinks(&listArg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -1146,7 +1149,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string dstPath := path.Join(f.slashRoot, dstRemote) // Check if destination exists - _, err := f.getDirMetadata(dstPath) + _, err := f.getDirMetadata(ctx, dstPath) if err == nil { return fs.ErrorDirExists } else if err != fs.ErrorDirNotFound { @@ -1165,7 +1168,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } err = f.pacer.Call(func() (bool, error) { _, err = f.srv.MoveV2(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return errors.Wrap(err, "MoveDir failed") @@ -1179,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { var q *users.SpaceUsage err = f.pacer.Call(func() (bool, error) { q, err = f.users.GetSpaceUsage() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "about failed") @@ -1210,7 +1213,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) { go func() { // get the StartCursor early so all changes from now on get processed - startCursor, err := f.changeNotifyCursor() + startCursor, err := f.changeNotifyCursor(ctx) if err != nil { fs.Infof(f, "Failed to get StartCursor: %s", err) } @@ -1235,7 +1238,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT } case <-tickerC: if startCursor == "" { - startCursor, err = f.changeNotifyCursor() + startCursor, err = f.changeNotifyCursor(ctx) if err != nil { fs.Infof(f, "Failed to get StartCursor: %s", err) continue @@ -1251,7 +1254,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT }() } -func (f *Fs) changeNotifyCursor() (cursor string, err error) { +func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) { var startCursor *files.ListFolderGetLatestCursorResult err = f.pacer.Call(func() (bool, error) { @@ -1266,7 +1269,7 @@ func (f *Fs) changeNotifyCursor() (cursor string, err error) { startCursor, err = f.srv.ListFolderGetLatestCursor(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -1296,7 +1299,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs. } res, err = f.svc.ListFolderLongpoll(&args) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -1319,7 +1322,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs. } err = f.pacer.Call(func() (bool, error) { changeList, err = f.srv.ListFolderContinue(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return "", errors.Wrap(err, "list continue") @@ -1392,7 +1395,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { if t != DbHashType { return "", hash.ErrUnsupported } - err := o.readMetaData() + err := o.readMetaData(ctx) if err != nil { return "", errors.Wrap(err, "failed to read hash from metadata") } @@ -1416,17 +1419,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error { } // Reads the entry for a file from dropbox -func (o *Object) readEntry() (*files.FileMetadata, error) { - return o.fs.getFileMetadata(o.remotePath()) +func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) { + return o.fs.getFileMetadata(ctx, o.remotePath()) } // Read entry if not set and set metadata from it -func (o *Object) readEntryAndSetMetadata() error { +func (o *Object) readEntryAndSetMetadata(ctx context.Context) error { // Last resort set time from client if !o.modTime.IsZero() { return nil } - entry, err := o.readEntry() + entry, err := o.readEntry(ctx) if err != nil { return err } @@ -1439,12 +1442,12 @@ func (o *Object) remotePath() string { } // readMetaData gets the info if it hasn't already been fetched -func (o *Object) readMetaData() (err error) { +func (o *Object) readMetaData(ctx context.Context) (err error) { if !o.modTime.IsZero() { return nil } // Last resort - return o.readEntryAndSetMetadata() + return o.readEntryAndSetMetadata(ctx) } // ModTime returns the modification time of the object @@ -1452,7 +1455,7 @@ func (o *Object) readMetaData() (err error) { // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { - err := o.readMetaData() + err := o.readMetaData(ctx) if err != nil { fs.Debugf(o, "Failed to read metadata: %v", err) return time.Now() @@ -1486,7 +1489,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { _, in, err = o.fs.sharing.GetSharedLinkFile(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -1502,7 +1505,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { _, in, err = o.fs.srv.Download(&arg) - return shouldRetry(err) + return shouldRetry(ctx, err) }) switch e := err.(type) { @@ -1521,7 +1524,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Will work optimally if size is >= uploadChunkSize. If the size is either // unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an // avoidable request to the Dropbox API that does not carry payload. -func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) { +func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) { chunkSize := int64(o.fs.opt.ChunkSize) chunks := 0 if size != -1 { @@ -1550,7 +1553,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size return false, nil } res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -1676,11 +1679,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var err error var entry *files.FileMetadata if size > int64(o.fs.opt.ChunkSize) || size == -1 { - entry, err = o.uploadChunked(in, commitInfo, size) + entry, err = o.uploadChunked(ctx, in, commitInfo, size) } else { err = o.fs.pacer.CallNoRetry(func() (bool, error) { entry, err = o.fs.srv.Upload(commitInfo, in) - return shouldRetry(err) + return shouldRetry(ctx, err) }) } if err != nil { @@ -1698,7 +1701,7 @@ func (o *Object) Remove(ctx context.Context) (err error) { _, err = o.fs.srv.DeleteV2(&files.DeleteArg{ Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()), }) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return err } diff --git a/backend/fichier/api.go b/backend/fichier/api.go index 8a9aa0754..991ad621a 100644 --- a/backend/fichier/api.go +++ b/backend/fichier/api.go @@ -28,7 +28,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } // Detect this error which the integration tests provoke // error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}" // @@ -74,7 +77,7 @@ func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) { var file File err := f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, &request, &file) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't read file info") @@ -96,7 +99,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons var token GetTokenResponse err := f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, &request, &token) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't list files") @@ -124,7 +127,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr var sharedFiles SharedFolderResponse err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't list files") @@ -153,7 +156,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi filesList = &FilesList{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't list files") @@ -181,7 +184,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol foldersList = &FoldersList{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't list folders") @@ -275,7 +278,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons response = &MakeFolderResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, &request, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't create folder") @@ -302,7 +305,7 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.rest.CallJSON(ctx, &opts, request, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't remove folder") @@ -331,7 +334,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes response = &GenericOKResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, request, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -358,7 +361,7 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri response = &MoveFileResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, request, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -383,7 +386,7 @@ func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename stri response = &CopyFileResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, request, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -405,7 +408,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse response = &GetUploadNodeResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, nil, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "didnt got an upload node") @@ -448,7 +451,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, err = f.pacer.CallNoRetry(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, nil, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -482,7 +485,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re response = &EndFileUploadResponse{} err = f.pacer.Call(func() (bool, error) { resp, err := f.rest.CallJSON(ctx, &opts, nil, response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { diff --git a/backend/fichier/object.go b/backend/fichier/object.go index 09ad7b29c..d48047035 100644 --- a/backend/fichier/object.go +++ b/backend/fichier/object.go @@ -94,7 +94,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.rest.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go index 4307f5b21..9bc95ddfa 100644 --- a/backend/filefabric/filefabric.go +++ b/backend/filefabric/filefabric.go @@ -228,7 +228,10 @@ var retryStatusCodes = []struct { // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } if err != nil { return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -401,7 +404,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr // Refresh the body each retry opts.Body = strings.NewReader(data.Encode()) resp, err = f.srv.CallJSON(ctx, &opts, nil, result) - return f.shouldRetry(resp, err, result) + return f.shouldRetry(ctx, resp, err, result) }) if err != nil { return resp, err @@ -1277,7 +1280,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader) - return o.fs.shouldRetry(resp, err, nil) + return o.fs.shouldRetry(ctx, resp, err, nil) }) if err != nil { return errors.Wrap(err, "failed to upload") diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index ffcde7258..3566e3b42 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -329,7 +329,10 @@ func (f *Fs) Features() *fs.Features { } // shouldRetry determines whether a given err rates being retried -func shouldRetry(err error) (again bool, errOut error) { +func shouldRetry(ctx context.Context, err error) (again bool, errOut error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } again = false if err != nil { if fserrors.ShouldRetry(err) { @@ -455,7 +458,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory) err = f.pacer.Call(func() (bool, error) { _, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err == nil { newRoot := path.Dir(f.root) @@ -521,7 +524,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck var objects *storage.Objects err = f.pacer.Call(func() (bool, error) { objects, err = list.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { if gErr, ok := err.(*googleapi.Error); ok { @@ -624,7 +627,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) var buckets *storage.Buckets err = f.pacer.Call(func() (bool, error) { buckets, err = listBuckets.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -750,7 +753,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) { // service account that only has the "Storage Object Admin" role. See #2193 for details. err = f.pacer.Call(func() (bool, error) { _, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err == nil { // Bucket already exists @@ -785,7 +788,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) { insertBucket.PredefinedAcl(f.opt.BucketACL) } _, err = insertBucket.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) }, nil) } @@ -802,7 +805,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) { return f.cache.Remove(bucket, func() error { return f.pacer.Call(func() (bool, error) { err = f.svc.Buckets.Delete(bucket).Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) }) } @@ -848,7 +851,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, for { err = f.pacer.Call(func() (bool, error) { rewriteResponse, err = rewriteRequest.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -941,7 +944,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er bucket, bucketPath := o.split() err = o.fs.pacer.Call(func() (bool, error) { object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { if gErr, ok := err.(*googleapi.Error); ok { @@ -1012,7 +1015,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL) } newObject, err = copyObject.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return err @@ -1043,7 +1046,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read _ = res.Body.Close() // ignore error } } - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -1109,7 +1112,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op insertObject.PredefinedAcl(o.fs.opt.ObjectACL) } newObject, err = insertObject.Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return err @@ -1124,7 +1127,7 @@ func (o *Object) Remove(ctx context.Context) (err error) { bucket, bucketPath := o.split() err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do() - return shouldRetry(err) + return shouldRetry(ctx, err) }) return err } diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index 4dc958593..35f58d09e 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -240,7 +240,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -329,7 +332,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e var openIDconfig map[string]interface{} err = f.pacer.Call(func() (bool, error) { resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", errors.Wrap(err, "couldn't read openID config") @@ -358,7 +361,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro } err = f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't read user info") @@ -389,7 +392,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) { var res interface{} err = f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, &res) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't revoke token") @@ -476,7 +479,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't list albums") @@ -531,7 +534,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't list files") @@ -675,7 +678,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, request, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't create album") @@ -810,7 +813,7 @@ func (o *Object) Size() int64 { } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { fs.Debugf(o, "Reading size failed: %v", err) @@ -861,7 +864,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't get media item") @@ -938,7 +941,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -993,10 +996,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) if err != nil { - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) } token, err = rest.ReadBody(resp) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't upload file") @@ -1024,7 +1027,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var result api.BatchCreateResponse err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to create media item") @@ -1069,7 +1072,7 @@ func (o *Object) Remove(ctx context.Context) (err error) { var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't delete item from album") diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 04678da8c..d970fdf04 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -235,7 +235,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -615,7 +618,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if apiErr, ok := err.(*api.Error); ok { @@ -854,7 +857,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &jf) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -883,7 +886,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e var result api.JottaFolder err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -995,7 +998,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { if apiErr, ok := err.(*api.Error); ok { @@ -1101,7 +1104,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "couldn't purge directory") @@ -1140,7 +1143,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1268,7 +1271,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var result api.JottaFile err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if apiErr, ok := err.(*api.Error); ok { @@ -1446,7 +1449,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1559,7 +1562,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var response api.AllocateFileResponse err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -1624,7 +1627,7 @@ func (o *Object) Remove(ctx context.Context) error { return o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 611cb005c..982ddc833 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -207,7 +207,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e Path: "/session/login.json", } resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to create session") @@ -294,7 +294,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error { Path: "/folder/remove.json", } resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) } @@ -389,7 +389,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, Path: "/file/move_copy.json", } resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -446,7 +446,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, Path: "/file/move_copy.json", } resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -495,7 +495,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string Path: "/folder/move_copy.json", } resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { fs.Debugf(src, "DirMove error %v", err) @@ -583,7 +583,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -633,7 +633,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . Path: "/upload/create_file.json", } resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to create file") @@ -659,7 +659,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -685,7 +688,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, Path: "/folder.json", } resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -713,7 +716,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID, } resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return "", false, errors.Wrap(err, "failed to get folder list") @@ -756,7 +759,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e folderList := FolderList{} err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to get folder list") @@ -845,7 +848,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { } err := o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) o.modTime = modTime @@ -865,7 +868,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to open file)") @@ -884,7 +887,7 @@ func (o *Object) Remove(ctx context.Context) error { Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id, } resp, err := o.fs.srv.Call(ctx, &opts) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) } @@ -913,7 +916,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Path: "/upload/open_file_upload.json", } resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to create file") @@ -957,7 +960,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to create file") @@ -980,7 +983,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Path: "/upload/close_file_upload.json", } resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to create file") @@ -1006,7 +1009,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Path: "/file/access.json", } resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -1032,7 +1035,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))), } resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to get folder list") diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 58734e18e..36caf7bf3 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -213,7 +213,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } doRetry := false // Check if it is an api.Error @@ -405,7 +408,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -460,7 +463,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -597,7 +600,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "rmdir failed") @@ -662,7 +665,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -700,7 +703,7 @@ func (f *Fs) CleanUp(ctx context.Context) error { return f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } @@ -740,7 +743,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -787,7 +790,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -814,7 +817,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str err := f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -838,7 +841,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str err = f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -869,7 +872,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &q) err = q.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "about failed") @@ -927,7 +930,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) { err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -1046,7 +1049,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) { err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -1072,7 +1075,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1151,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { // sometimes pcloud leaves a half complete file on @@ -1181,7 +1184,7 @@ func (o *Object) Remove(ctx context.Context) error { return o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index eca5cd7f7..76c512cec 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -176,7 +176,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -370,7 +373,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -407,7 +410,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -581,7 +584,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var result api.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "rmdir failed") @@ -660,7 +663,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir var result api.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "Move http") @@ -769,7 +772,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "CreateDir http") @@ -896,7 +899,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -934,7 +937,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) if err != nil { - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) } // Just check the download URL resolves - sometimes // the URLs returned by premiumize.me don't resolve so @@ -993,7 +996,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var result api.Response err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "upload file http") @@ -1035,7 +1038,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str var result api.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "rename http") @@ -1060,7 +1063,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) { var result api.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "remove http") diff --git a/backend/putio/error.go b/backend/putio/error.go index 3fdfc2f15..34eecabb9 100644 --- a/backend/putio/error.go +++ b/backend/putio/error.go @@ -1,6 +1,7 @@ package putio import ( + "context" "fmt" "net/http" @@ -29,7 +30,10 @@ func (e *statusCodeError) Temporary() bool { // shouldRetry returns a boolean as to whether this err deserves to be // retried. It returns the err as a convenience -func shouldRetry(err error) (bool, error) { +func shouldRetry(ctx context.Context, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } if err == nil { return false, nil } diff --git a/backend/putio/fs.go b/backend/putio/fs.go index f0cf4827a..fc8e352ac 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID) entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return itoa(entry.ID), err } @@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "listing file: %d", fileID) children, _, err = f.client.Files.List(ctx, fileID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 { @@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "listing files inside List: %d", parentID) children, _, err = f.client.Files.List(ctx, parentID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "getting file: %d", fileID) entry, err = f.client.Files.Get(ctx, fileID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -295,7 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt)) fs.OpenOptionAddHTTPHeaders(req.Header, options) resp, err := f.oAuthClient.Do(req) - retry, err := shouldRetry(err) + retry, err := shouldRetry(ctx, err) if retry { return true, err } @@ -320,7 +320,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io. err = f.pacer.Call(func() (bool, error) { fs.Debugf(f, "Sending zero length chunk") _, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0) - return shouldRetry(err) + return shouldRetry(ctx, err) }) return } @@ -344,13 +344,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io. // Get file offset and seek to the position offset, err := f.getServerOffset(ctx, location) if err != nil { - return shouldRetry(err) + return shouldRetry(ctx, err) } sentBytes := offset - chunkStart fs.Debugf(f, "sentBytes: %d", sentBytes) _, err = chunk.Seek(sentBytes, io.SeekStart) if err != nil { - return shouldRetry(err) + return shouldRetry(ctx, err) } transferOffset = offset reqSize = chunkSize - sentBytes @@ -367,7 +367,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io. offsetMismatch = true return true, errors.New("connection broken") } - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -479,7 +479,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "listing files: %d", dirID) children, _, err = f.client.Files.List(ctx, dirID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return errors.Wrap(err, "Rmdir") @@ -493,7 +493,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "deleting file: %d", dirID) err = f.client.Files.Delete(ctx, dirID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) f.dirCache.FlushDir(dir) return err @@ -552,7 +552,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec req.Header.Set("Content-Type", "application/x-www-form-urlencoded") // fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID) _, err = f.client.Do(req, nil) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -591,7 +591,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec req.Header.Set("Content-Type", "application/x-www-form-urlencoded") // fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID) _, err = f.client.Do(req, nil) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -631,7 +631,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string req.Header.Set("Content-Type", "application/x-www-form-urlencoded") // fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID) _, err = f.client.Do(req, nil) - return shouldRetry(err) + return shouldRetry(ctx, err) }) srcFs.dirCache.FlushDir(srcRemote) return err @@ -644,7 +644,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { err = f.pacer.Call(func() (bool, error) { // fs.Debugf(f, "getting account info") ai, err = f.client.Account.Info(ctx) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, errors.Wrap(err, "about failed") @@ -678,6 +678,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { } // fs.Debugf(f, "emptying trash") _, err = f.client.Do(req, nil) - return shouldRetry(err) + return shouldRetry(ctx, err) }) } diff --git a/backend/putio/object.go b/backend/putio/object.go index f653ffb0b..a4a1e8416 100644 --- a/backend/putio/object.go +++ b/backend/putio/object.go @@ -145,7 +145,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) { if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 { return false, fs.ErrorObjectNotFound } - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return nil, err @@ -220,7 +220,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var storageURL string err = o.fs.pacer.Call(func() (bool, error) { storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if err != nil { return @@ -231,7 +231,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read err = o.fs.pacer.Call(func() (bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil) if err != nil { - return shouldRetry(err) + return shouldRetry(ctx, err) } req.Header.Set("User-Agent", o.fs.client.UserAgent) @@ -241,7 +241,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } // fs.Debugf(o, "opening file: id=%d", o.file.ID) resp, err = o.fs.httpClient.Do(req) - return shouldRetry(err) + return shouldRetry(ctx, err) }) if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 { _ = resp.Body.Close() @@ -283,6 +283,6 @@ func (o *Object) Remove(ctx context.Context) (err error) { return o.fs.pacer.Call(func() (bool, error) { // fs.Debugf(o, "removing file: id=%d", o.file.ID) err = o.fs.client.Files.Delete(ctx, o.file.ID) - return shouldRetry(err) + return shouldRetry(ctx, err) }) } diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index e96d1575a..794efd856 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -408,7 +408,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } // For 429 errors look at the Retry-After: header and // set the retry appropriately, starting with a minimum of 1 // second if it isn't set. diff --git a/backend/seafile/webapi.go b/backend/seafile/webapi.go index 8c067ac6c..f91a9ae2b 100644 --- a/backend/seafile/webapi.go +++ b/backend/seafile/webapi.go @@ -86,7 +86,7 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -112,7 +112,7 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo, var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -139,7 +139,7 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -170,7 +170,7 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -197,7 +197,7 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -228,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -271,7 +271,7 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -316,7 +316,7 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string) var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -358,7 +358,7 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -398,7 +398,7 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string) var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -438,7 +438,7 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -474,7 +474,7 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -505,7 +505,7 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -539,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error { } err := f.pacer.Call(func() (bool, error) { resp, err := f.srv.CallJSON(ctx, &opts, nil, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to delete file") @@ -565,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -614,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -659,7 +659,7 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -739,7 +739,7 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -777,7 +777,7 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -818,7 +818,7 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID, var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -860,7 +860,7 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID, var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -900,7 +900,7 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &request, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -938,7 +938,7 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -976,7 +976,7 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -1030,7 +1030,7 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { @@ -1075,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil { diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index e7b6e53f1..5a160052c 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -299,7 +299,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -324,7 +327,7 @@ func (f *Fs) readMetaDataForIDPath(ctx context.Context, id, path string, directo var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &item) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { @@ -631,7 +634,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &req, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", errors.Wrap(err, "CreateDir") @@ -663,7 +666,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -912,7 +915,7 @@ func (f *Fs) updateItem(ctx context.Context, id, leaf, directoryID string, modTi var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &update, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1133,7 +1136,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj var info *api.Item err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1294,7 +1297,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var dl api.DownloadSpecification err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "open: fetch download specification") @@ -1309,7 +1312,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "open") @@ -1365,7 +1368,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "upload get specification") @@ -1390,7 +1393,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var finish api.UploadFinishResponse err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "upload file") @@ -1426,7 +1429,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) { var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "remove") diff --git a/backend/sharefile/upload.go b/backend/sharefile/upload.go index 12218ab96..f6ee40c86 100644 --- a/backend/sharefile/upload.go +++ b/backend/sharefile/upload.go @@ -155,7 +155,7 @@ func (up *largeUpload) finish(ctx context.Context) error { err := up.f.pacer.Call(func() (bool, error) { resp, err := up.f.srv.Call(ctx, &opts) if err != nil { - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) } respBody, err = rest.ReadBody(resp) // retry all errors now that the multipart upload has started diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index 32335af28..aba130eb9 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -111,7 +111,7 @@ func init() { // FIXME //err = f.pacer.Call(func() (bool, error) { resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil) - // return shouldRetry(resp, err) + // return shouldRetry(ctx, resp, err) //}) if err != nil { log.Fatalf("Failed to get token: %v", err) @@ -248,7 +248,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -288,7 +291,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { @@ -325,7 +328,7 @@ func (f *Fs) getAuthToken(ctx context.Context) error { } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to get authorization") @@ -373,7 +376,7 @@ func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) { } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &user) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to get user") @@ -567,7 +570,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -618,7 +621,7 @@ OUTER: var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -774,7 +777,7 @@ func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string, } return f.pacer.Call(func() (bool, error) { resp, err := f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } // Move file/dir to deleted files if not hard delete @@ -880,7 +883,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, ©File, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -934,7 +937,7 @@ func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info * var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, &move, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -964,7 +967,7 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err var resp *http.Response return f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, &move, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) } @@ -1053,7 +1056,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var info *api.File err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -1182,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1204,7 +1207,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return "", err @@ -1262,7 +1265,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to upload file") diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index dfc490947..ce2fb3ae1 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -196,7 +196,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } // If we have a bearer token command and it has expired then refresh it if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 { fs.Debugf(f, "Bearer token expired: %v", err) @@ -270,7 +273,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if apiErr, ok := err.(*api.Error); ok { // does not exist @@ -628,7 +631,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if apiErr, ok := err.(*api.Error); ok { @@ -800,7 +803,7 @@ func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &result) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) return err == nil } @@ -822,7 +825,7 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error { } err := f.pacer.Call(func() (bool, error) { resp, err := f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if apiErr, ok := err.(*api.Error); ok { // Check if it already exists. The response code for this isn't @@ -911,7 +914,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, nil) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "rmdir failed") @@ -974,7 +977,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "Copy call failed") @@ -1070,7 +1073,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "DirMove MOVE call failed") @@ -1112,7 +1115,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallXML(ctx, &opts, nil, &q) - return f.shouldRetry(resp, err) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "about call failed") @@ -1240,7 +1243,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1291,7 +1294,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } err = o.fs.pacer.CallNoRetry(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { // Give the WebDAV server a chance to get its internal state in order after the @@ -1318,7 +1321,7 @@ func (o *Object) Remove(ctx context.Context) error { } return o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.Call(ctx, &opts) - return o.fs.shouldRetry(resp, err) + return o.fs.shouldRetry(ctx, resp, err) }) } diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 1ad22f784..3c850932e 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -153,7 +153,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } @@ -226,7 +229,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api. var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -468,7 +471,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) { err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { // fmt.Printf("CreateDir %q Error: %s\n", path, err.Error()) @@ -543,6 +546,9 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) { var body []byte err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) + if fserrors.ContextError(ctx, &err) { + return false, err + } if err != nil { return fserrors.ShouldRetry(err), err } @@ -585,6 +591,9 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro var body []byte err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) + if fserrors.ContextError(ctx, &err) { + return false, err + } if err != nil { return fserrors.ShouldRetry(err), err } @@ -658,6 +667,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite var body []byte err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) + if fserrors.ContextError(ctx, &err) { + return false, err + } if err != nil { return fserrors.ShouldRetry(err), err } @@ -810,7 +822,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if apiErr, ok := err.(*api.ErrorResponse); ok { @@ -848,7 +860,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return err } @@ -865,7 +877,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -999,7 +1011,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return err } @@ -1032,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1047,7 +1059,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1071,7 +1083,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1089,7 +1101,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) return err diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 50a240fa1..87f0df8a1 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -257,7 +257,10 @@ var retryErrorCodes = []int{ // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func shouldRetry(resp *http.Response, err error) (bool, error) { +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } authRetry := false if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { @@ -354,7 +357,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error var err error err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -450,7 +453,7 @@ OUTER: var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") @@ -555,7 +558,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) @@ -664,7 +667,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, var uploadResponse *api.UploadResponse err = f.pacer.CallNoRetry(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "upload error") @@ -746,7 +749,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) (err error) { } err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "delete object failed") @@ -816,7 +819,7 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e var result *api.ItemInfo err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "rename failed") @@ -869,7 +872,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var result *api.ItemList err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, ©File, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "couldn't copy file") @@ -914,7 +917,7 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item, var result *api.ItemList err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "move failed") @@ -1181,7 +1184,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(ctx, &opts) - return shouldRetry(resp, err) + return shouldRetry(ctx, resp, err) }) if err != nil { return nil, err diff --git a/fs/fserrors/error.go b/fs/fserrors/error.go index 76922e55c..b3fbfcce6 100644 --- a/fs/fserrors/error.go +++ b/fs/fserrors/error.go @@ -2,6 +2,7 @@ package fserrors import ( + "context" "fmt" "io" "net/http" @@ -437,6 +438,22 @@ func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool { return false } +// ContextError checks to see if ctx is in error. +// +// If it is in error then it overwrites *perr with the context error +// if *perr was nil and returns true. +// +// Otherwise it returns false. +func ContextError(ctx context.Context, perr *error) bool { + if ctxErr := ctx.Err(); ctxErr != nil { + if *perr == nil { + *perr = ctxErr + } + return true + } + return false +} + type causer interface { Cause() error } diff --git a/fs/fserrors/error_test.go b/fs/fserrors/error_test.go index f153c7aab..da8391b0e 100644 --- a/fs/fserrors/error_test.go +++ b/fs/fserrors/error_test.go @@ -1,6 +1,7 @@ package fserrors import ( + "context" "fmt" "io" "net" @@ -170,3 +171,21 @@ func TestRetryAfter(t *testing.T) { assert.True(t, IsRetryAfterError(err)) assert.Contains(t, e.Error(), "try again after") } + +func TestContextError(t *testing.T) { + var err = io.EOF + ctx, cancel := context.WithCancel(context.Background()) + + assert.False(t, ContextError(ctx, &err)) + assert.Equal(t, io.EOF, err) + + cancel() + + assert.True(t, ContextError(ctx, &err)) + assert.Equal(t, io.EOF, err) + + err = nil + + assert.True(t, ContextError(ctx, &err)) + assert.Equal(t, context.Canceled, err) +} diff --git a/fs/operations/check.go b/fs/operations/check.go index 4f095e587..20017707f 100644 --- a/fs/operations/check.go +++ b/fs/operations/check.go @@ -311,7 +311,7 @@ func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) { // it returns true if differences were found func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) { ci := fs.GetConfig(ctx) - err = Retry(src, ci.LowLevelRetries, func() error { + err = Retry(ctx, src, ci.LowLevelRetries, func() error { differ, err = checkIdenticalDownload(ctx, dst, src) return err }) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index ab6381611..822c4286e 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -485,6 +485,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj break } // Retry if err returned a retry error + if fserrors.ContextError(ctx, &err) { + break + } var retry bool if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { retry = true @@ -800,7 +803,7 @@ func SameDir(fdst, fsrc fs.Info) bool { } // Retry runs fn up to maxTries times if it returns a retriable error -func Retry(o interface{}, maxTries int, fn func() error) (err error) { +func Retry(ctx context.Context, o interface{}, maxTries int, fn func() error) (err error) { for tries := 1; tries <= maxTries; tries++ { // Call the function which might error err = fn() @@ -808,6 +811,9 @@ func Retry(o interface{}, maxTries int, fn func() error) (err error) { break } // Retry if err returned a retry error + if fserrors.ContextError(ctx, &err) { + break + } if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries) continue diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 76172952b..4fb90bee9 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -405,6 +405,8 @@ func TestDelete(t *testing.T) { } func TestRetry(t *testing.T) { + ctx := context.Background() + var i int var err error fn := func() error { @@ -416,15 +418,15 @@ func TestRetry(t *testing.T) { } i, err = 3, io.EOF - assert.Equal(t, nil, operations.Retry(nil, 5, fn)) + assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 0, i) i, err = 10, io.EOF - assert.Equal(t, io.EOF, operations.Retry(nil, 5, fn)) + assert.Equal(t, io.EOF, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 5, i) i, err = 10, fs.ErrorObjectNotFound - assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(nil, 5, fn)) + assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(ctx, nil, 5, fn)) assert.Equal(t, 9, i) }