mirror of
https://github.com/rclone/rclone
synced 2024-11-14 13:36:24 +01:00
amazonclouddrive: remove Amazon Drive backend code and docs #7539
The Amazon Drive backend is closed from 2023-12-31. See: https://www.amazon.com/b?ie=UTF8&node=23943055011
This commit is contained in:
Nick Craig-Wood
parent
578b9df6ea
commit
0e746f25a3
@ -23,7 +23,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
|
||||
* Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
|
||||
* Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
* Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
|
||||
* Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
* ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
* Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
|
||||
|
||||
@ -4,7 +4,6 @@ package all
|
||||
import (
|
||||
// Active file systems
|
||||
_ "github.com/rclone/rclone/backend/alias"
|
||||
_ "github.com/rclone/rclone/backend/amazonclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/azureblob"
|
||||
_ "github.com/rclone/rclone/backend/azurefiles"
|
||||
_ "github.com/rclone/rclone/backend/b2"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,21 +0,0 @@
|
||||
// Test AmazonCloudDrive filesystem interface
|
||||
|
||||
//go:build acd
|
||||
// +build acd
|
||||
|
||||
package amazonclouddrive_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/amazonclouddrive"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
|
||||
fstests.RemoteName = "TestAmazonCloudDrive:"
|
||||
fstests.Run(t)
|
||||
}
|
||||
@ -23,8 +23,6 @@ CATEGORY = re.compile(r"(^[\w/ ]+(?:, *[\w/ ]+)*):\s*(.*)$")
|
||||
backends = [ x for x in os.listdir("backend") if x != "all"]
|
||||
|
||||
backend_aliases = {
|
||||
"amazon cloud drive" : "amazonclouddrive",
|
||||
"acd" : "amazonclouddrive",
|
||||
"google cloud storage" : "googlecloudstorage",
|
||||
"gcs" : "googlecloudstorage",
|
||||
"azblob" : "azureblob",
|
||||
@ -34,7 +32,6 @@ backend_aliases = {
|
||||
}
|
||||
|
||||
backend_titles = {
|
||||
"amazonclouddrive": "Amazon Cloud Drive",
|
||||
"googlecloudstorage": "Google Cloud Storage",
|
||||
"azureblob": "Azure Blob",
|
||||
"ftp": "FTP",
|
||||
|
||||
@ -30,7 +30,6 @@ docs = [
|
||||
# Keep these alphabetical by full name
|
||||
"fichier.md",
|
||||
"alias.md",
|
||||
"amazonclouddrive.md",
|
||||
"s3.md",
|
||||
"b2.md",
|
||||
"box.md",
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
exec rclone --check-normalization=true --check-control=true --check-length=true info \
|
||||
/tmp/testInfo \
|
||||
TestAmazonCloudDrive:testInfo \
|
||||
TestB2:testInfo \
|
||||
TestCryptDrive:testInfo \
|
||||
TestCryptSwift:testInfo \
|
||||
|
||||
@ -8,7 +8,6 @@ export PATH=$GOPATH/src/github.com/rclone/rclone:$PATH
|
||||
|
||||
typeset -A allRemotes
|
||||
allRemotes=(
|
||||
TestAmazonCloudDrive '--low-level-retries=2 --checkers=5 --upload-wait=5s'
|
||||
TestB2 ''
|
||||
TestBox ''
|
||||
TestDrive '--tpslimit=5'
|
||||
|
||||
@ -107,7 +107,6 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="1Fichier" home="https://1fichier.com/" config="/fichier/" start="true">}}
|
||||
{{< provider name="Akamai Netstorage" home="https://www.akamai.com/us/en/products/media-delivery/netstorage.jsp" config="/netstorage/" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" note="#status">}}
|
||||
{{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
|
||||
{{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}
|
||||
{{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
|
||||
|
||||
@ -2,339 +2,15 @@
|
||||
title: "Amazon Drive"
|
||||
description: "Rclone docs for Amazon Drive"
|
||||
versionIntroduced: "v1.20"
|
||||
status: Deprecated
|
||||
---
|
||||
|
||||
# {{< icon "fab fa-amazon" >}} Amazon Drive
|
||||
|
||||
Amazon Drive, formerly known as Amazon Cloud Drive, is a cloud storage
|
||||
Amazon Drive, formerly known as Amazon Cloud Drive, was a cloud storage
|
||||
service run by Amazon for consumers.
|
||||
|
||||
## Status
|
||||
|
||||
**Important:** rclone supports Amazon Drive only if you have your own
|
||||
set of API keys. Unfortunately the [Amazon Drive developer
|
||||
program](https://developer.amazon.com/amazon-drive) is now closed to
|
||||
new entries so if you don't already have your own set of keys you will
|
||||
not be able to use rclone with Amazon Drive.
|
||||
|
||||
For the history on why rclone no longer has a set of Amazon Drive API
|
||||
keys see [the forum](https://forum.rclone.org/t/rclone-has-been-banned-from-amazon-drive/2314).
|
||||
|
||||
If you happen to know anyone who works at Amazon then please ask them
|
||||
to re-instate rclone into the Amazon Drive developer program - thanks!
|
||||
|
||||
## Configuration
|
||||
|
||||
The initial setup for Amazon Drive involves getting a token from
|
||||
Amazon which you need to do in your browser. `rclone config` walks
|
||||
you through it.
|
||||
|
||||
The configuration process for Amazon Drive may involve using an [oauth
|
||||
proxy](https://github.com/ncw/oauthproxy). This is used to keep the
|
||||
Amazon credentials out of the source code. The proxy runs in Google's
|
||||
very secure App Engine environment and doesn't store any credentials
|
||||
which pass through it.
|
||||
|
||||
Since rclone doesn't currently have its own Amazon Drive credentials
|
||||
so you will either need to have your own `client_id` and
|
||||
`client_secret` with Amazon Drive, or use a third-party oauth proxy
|
||||
in which case you will need to enter `client_id`, `client_secret`,
|
||||
`auth_url` and `token_url`.
|
||||
|
||||
Note also if you are not using Amazon's `auth_url` and `token_url`,
|
||||
(ie you filled in something for those) then if setting up on a remote
|
||||
machine you can only use the [copying the config method of
|
||||
configuration](https://rclone.org/remote_setup/#configuring-by-copying-the-config-file)
|
||||
- `rclone authorize` will not work.
|
||||
|
||||
Here is an example of how to make a remote called `remote`. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
r) Rename remote
|
||||
c) Copy remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/r/c/s/q> n
|
||||
name> remote
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
[snip]
|
||||
Storage> amazon cloud drive
|
||||
Amazon Application Client Id - required.
|
||||
client_id> your client ID goes here
|
||||
Amazon Application Client Secret - required.
|
||||
client_secret> your client secret goes here
|
||||
Auth server URL - leave blank to use Amazon's.
|
||||
auth_url> Optional auth URL
|
||||
Token server url - leave blank to use Amazon's.
|
||||
token_url> Optional token URL
|
||||
Remote config
|
||||
Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config.
|
||||
Use web browser to automatically authenticate rclone with remote?
|
||||
* Say Y if the machine running rclone has a web browser you can use
|
||||
* Say N if running rclone on a (remote) machine without web browser access
|
||||
If not sure try Y. If Y failed, try N.
|
||||
y) Yes
|
||||
n) No
|
||||
y/n> y
|
||||
If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
|
||||
Log in and authorize rclone for access
|
||||
Waiting for code...
|
||||
Got code
|
||||
--------------------
|
||||
[remote]
|
||||
client_id = your client ID goes here
|
||||
client_secret = your client secret goes here
|
||||
auth_url = Optional auth URL
|
||||
token_url = Optional token URL
|
||||
token = {"access_token":"xxxxxxxxxxxxxxxxxxxxxxx","token_type":"bearer","refresh_token":"xxxxxxxxxxxxxxxxxx","expiry":"2015-09-06T16:07:39.658438471+01:00"}
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Amazon. This only runs from the moment it
|
||||
opens your browser to the moment you get back the verification
|
||||
code. This is on `http://127.0.0.1:53682/` and this it may require
|
||||
you to unblock it temporarily if you are running a host firewall.
|
||||
|
||||
Once configured you can then use `rclone` like this,
|
||||
|
||||
List directories in top level of your Amazon Drive
|
||||
|
||||
rclone lsd remote:
|
||||
|
||||
List all the files in your Amazon Drive
|
||||
|
||||
rclone ls remote:
|
||||
|
||||
To copy a local directory to an Amazon Drive directory called backup
|
||||
|
||||
rclone copy /home/source remote:backup
|
||||
|
||||
### Modification times and hashes
|
||||
|
||||
Amazon Drive doesn't allow modification times to be changed via
|
||||
the API so these won't be accurate or used for syncing.
|
||||
|
||||
It does support the MD5 hash algorithm, so for a more accurate sync,
|
||||
you can use the `--checksum` flag.
|
||||
|
||||
### Restricted filename characters
|
||||
|
||||
| Character | Value | Replacement |
|
||||
| --------- |:-----:|:-----------:|
|
||||
| NUL | 0x00 | ␀ |
|
||||
| / | 0x2F | ／ |
|
||||
|
||||
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
|
||||
as they can't be used in JSON strings.
|
||||
|
||||
### Deleting files
|
||||
|
||||
Any files you delete with rclone will end up in the trash. Amazon
|
||||
don't provide an API to permanently delete files, nor to empty the
|
||||
trash, so you will have to do that with one of Amazon's apps or via
|
||||
the Amazon Drive website. As of November 17, 2016, files are
|
||||
automatically deleted by Amazon from the trash after 30 days.
|
||||
|
||||
### Using with non `.com` Amazon accounts
|
||||
|
||||
Let's say you usually use `amazon.co.uk`. When you authenticate with
|
||||
rclone it will take you to an `amazon.com` page to log in. Your
|
||||
`amazon.co.uk` email and password should work here just fine.
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/amazonclouddrive/amazonclouddrive.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the Standard options specific to amazon cloud drive (Amazon Drive).
|
||||
|
||||
#### --acd-client-id
|
||||
|
||||
OAuth Client Id.
|
||||
|
||||
Leave blank normally.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_id
|
||||
- Env Var: RCLONE_ACD_CLIENT_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --acd-client-secret
|
||||
|
||||
OAuth Client Secret.
|
||||
|
||||
Leave blank normally.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_secret
|
||||
- Env Var: RCLONE_ACD_CLIENT_SECRET
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the Advanced options specific to amazon cloud drive (Amazon Drive).
|
||||
|
||||
#### --acd-token
|
||||
|
||||
OAuth Access Token as a JSON blob.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: token
|
||||
- Env Var: RCLONE_ACD_TOKEN
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --acd-auth-url
|
||||
|
||||
Auth server URL.
|
||||
|
||||
Leave blank to use the provider defaults.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: auth_url
|
||||
- Env Var: RCLONE_ACD_AUTH_URL
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --acd-token-url
|
||||
|
||||
Token server url.
|
||||
|
||||
Leave blank to use the provider defaults.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: token_url
|
||||
- Env Var: RCLONE_ACD_TOKEN_URL
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --acd-checkpoint
|
||||
|
||||
Checkpoint for internal polling (debug).
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: checkpoint
|
||||
- Env Var: RCLONE_ACD_CHECKPOINT
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --acd-upload-wait-per-gb
|
||||
|
||||
Additional time per GiB to wait after a failed complete upload to see if it appears.
|
||||
|
||||
Sometimes Amazon Drive gives an error when a file has been fully
|
||||
uploaded but the file appears anyway after a little while. This
|
||||
happens sometimes for files over 1 GiB in size and nearly every time for
|
||||
files bigger than 10 GiB. This parameter controls the time rclone waits
|
||||
for the file to appear.
|
||||
|
||||
The default value for this parameter is 3 minutes per GiB, so by
|
||||
default it will wait 3 minutes for every GiB uploaded to see if the
|
||||
file appears.
|
||||
|
||||
You can disable this feature by setting it to 0. This may cause
|
||||
conflict errors as rclone retries the failed upload but the file will
|
||||
most likely appear correctly eventually.
|
||||
|
||||
These values were determined empirically by observing lots of uploads
|
||||
of big files for a range of file sizes.
|
||||
|
||||
Upload with the "-v" flag to see more info about what rclone is doing
|
||||
in this situation.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: upload_wait_per_gb
|
||||
- Env Var: RCLONE_ACD_UPLOAD_WAIT_PER_GB
|
||||
- Type: Duration
|
||||
- Default: 3m0s
|
||||
|
||||
#### --acd-templink-threshold
|
||||
|
||||
Files >= this size will be downloaded via their tempLink.
|
||||
|
||||
Files this size or more will be downloaded via their "tempLink". This
|
||||
is to work around a problem with Amazon Drive which blocks downloads
|
||||
of files bigger than about 10 GiB. The default for this is 9 GiB which
|
||||
shouldn't need to be changed.
|
||||
|
||||
To download files above this threshold, rclone requests a "tempLink"
|
||||
which downloads the file through a temporary URL directly from the
|
||||
underlying S3 storage.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: templink_threshold
|
||||
- Env Var: RCLONE_ACD_TEMPLINK_THRESHOLD
|
||||
- Type: SizeSuffix
|
||||
- Default: 9Gi
|
||||
|
||||
#### --acd-encoding
|
||||
|
||||
The encoding for the backend.
|
||||
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: encoding
|
||||
- Env Var: RCLONE_ACD_ENCODING
|
||||
- Type: Encoding
|
||||
- Default: Slash,InvalidUtf8,Dot
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
## Limitations
|
||||
|
||||
Note that Amazon Drive is case insensitive so you can't have a
|
||||
file called "Hello.doc" and one called "hello.doc".
|
||||
|
||||
Amazon Drive has rate limiting so you may notice errors in the
|
||||
sync (429 errors). rclone will automatically retry the sync up to 3
|
||||
times by default (see `--retries` flag) which should hopefully work
|
||||
around this problem.
|
||||
|
||||
Amazon Drive has an internal limit of file sizes that can be uploaded
|
||||
to the service. This limit is not officially published, but all files
|
||||
larger than this will fail.
|
||||
|
||||
At the time of writing (Jan 2016) is in the area of 50 GiB per file.
|
||||
This means that larger files are likely to fail.
|
||||
|
||||
Unfortunately there is no way for rclone to see that this failure is
|
||||
because of file size, so it will retry the operation, as any other
|
||||
failure. To avoid this problem, use `--max-size 50000M` option to limit
|
||||
the maximum size of uploaded files. Note that `--max-size` does not split
|
||||
files into segments, it only ignores files over this size.
|
||||
|
||||
`rclone about` is not supported by the Amazon Drive backend. Backends without
|
||||
this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
From 2023-12-31, [Amazon Drive has been discontinued](https://www.amazon.com/b?ie=UTF8&node=23943055011)
|
||||
by Amazon so the Amazon Drive backend has been removed.
|
||||
|
||||
You can still use Amazon Photos to access your photos and videos.
|
||||
|
||||
@ -784,7 +784,7 @@ encoding is modified in two ways:
|
||||
* we strip the padding character `=`
|
||||
|
||||
`base32` is used rather than the more efficient `base64` so rclone can be
|
||||
used on case insensitive remotes (e.g. Windows, Amazon Drive).
|
||||
used on case insensitive remotes (e.g. Windows, Box, Dropbox, Onedrive etc).
|
||||
|
||||
### Key derivation
|
||||
|
||||
|
||||
@ -30,7 +30,6 @@ See the following for detailed instructions for
|
||||
* [1Fichier](/fichier/)
|
||||
* [Akamai Netstorage](/netstorage/)
|
||||
* [Alias](/alias/)
|
||||
* [Amazon Drive](/amazonclouddrive/)
|
||||
* [Amazon S3](/s3/)
|
||||
* [Backblaze B2](/b2/)
|
||||
* [Box](/box/)
|
||||
|
||||
@ -18,7 +18,6 @@ Here is an overview of the major features of each cloud storage system.
|
||||
| ---------------------------- |:-----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:|
|
||||
| 1Fichier | Whirlpool | - | No | Yes | R | - |
|
||||
| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - |
|
||||
| Amazon Drive | MD5 | - | Yes | No | R | - |
|
||||
| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU |
|
||||
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
|
||||
| Box | SHA1 | R/W | Yes | No | - | - |
|
||||
@ -471,7 +470,6 @@ upon backend-specific capabilities.
|
||||
| ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------------|:------------:|:-----:|:--------:|
|
||||
| 1Fichier | No | Yes | Yes | No | No | No | No | No | Yes | No | Yes |
|
||||
| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | No | Yes |
|
||||
| Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
|
||||
| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
|
||||
|
||||
@ -36,7 +36,7 @@ For more help and alternate methods see: https://rclone.org/remote_setup/
|
||||
Execute the following on the machine with the web browser (same rclone
|
||||
version recommended):
|
||||
|
||||
rclone authorize "amazon cloud drive"
|
||||
rclone authorize "dropbox"
|
||||
|
||||
Then paste the result below:
|
||||
result>
|
||||
@ -45,7 +45,7 @@ result>
|
||||
Then on your main desktop machine
|
||||
|
||||
```
|
||||
rclone authorize "amazon cloud drive"
|
||||
rclone authorize "dropbox"
|
||||
If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
|
||||
Log in and authorize rclone for access
|
||||
Waiting for code...
|
||||
|
||||
@ -84,7 +84,7 @@ name> remote
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -2452,10 +2452,10 @@ Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> 5
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
@ -2576,18 +2576,11 @@ To configure access to IBM COS S3, follow the steps below:
|
||||
3. Select "s3" storage.
|
||||
```
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Alias for an existing remote
|
||||
\ "alias"
|
||||
2 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS)
|
||||
\ "s3"
|
||||
4 / Backblaze B2
|
||||
\ "b2"
|
||||
[snip]
|
||||
23 / HTTP
|
||||
\ "http"
|
||||
Storage> 3
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
```
|
||||
|
||||
4. Select IBM COS as the S3 Storage Provider.
|
||||
@ -2747,7 +2740,7 @@ Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -2853,7 +2846,7 @@ Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -3091,15 +3084,8 @@ name> qiniu
|
||||
|
||||
```
|
||||
Choose a number from below, or type in your own value
|
||||
1 / 1Fichier
|
||||
\ (fichier)
|
||||
2 / Akamai NetStorage
|
||||
\ (netstorage)
|
||||
3 / Alias for an existing remote
|
||||
\ (alias)
|
||||
4 / Amazon Drive
|
||||
\ (amazon cloud drive)
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -3364,7 +3350,7 @@ Choose `s3` backend
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -3665,7 +3651,7 @@ Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -3775,7 +3761,7 @@ Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
...
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
...
|
||||
Storage> s3
|
||||
@ -4031,15 +4017,8 @@ name> leviia
|
||||
|
||||
```
|
||||
Choose a number from below, or type in your own value
|
||||
1 / 1Fichier
|
||||
\ (fichier)
|
||||
2 / Akamai NetStorage
|
||||
\ (netstorage)
|
||||
3 / Alias for an existing remote
|
||||
\ (alias)
|
||||
4 / Amazon Drive
|
||||
\ (amazon cloud drive)
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -4252,7 +4231,7 @@ Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
X / Amazon S3 Compliant Storage Providers including AWS, ...Linode, ...and others
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...Linode, ...and others
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -4497,13 +4476,8 @@ name> cos
|
||||
|
||||
```
|
||||
Choose a number from below, or type in your own value
|
||||
1 / 1Fichier
|
||||
\ "fichier"
|
||||
2 / Alias for an existing remote
|
||||
\ "alias"
|
||||
3 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@ -4911,7 +4885,7 @@ Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ "s3"
|
||||
|
||||
Storage> s3
|
||||
|
||||
@ -54,7 +54,6 @@
|
||||
<a class="dropdown-item" href="/fichier/"><i class="fa fa-archive fa-fw"></i> 1Fichier</a>
|
||||
<a class="dropdown-item" href="/netstorage/"><i class="fas fa-database fa-fw"></i> Akamai NetStorage</a>
|
||||
<a class="dropdown-item" href="/alias/"><i class="fa fa-link fa-fw"></i> Alias</a>
|
||||
<a class="dropdown-item" href="/amazonclouddrive/"><i class="fab fa-amazon fa-fw"></i> Amazon Drive</a>
|
||||
<a class="dropdown-item" href="/s3/"><i class="fab fa-amazon fa-fw"></i> Amazon S3</a>
|
||||
<a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a>
|
||||
<a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a>
|
||||
|
||||
@ -284,7 +284,7 @@ func CheckListingWithRoot(t *testing.T, f fs.Fs, dir string, items []Item, expec
|
||||
listingOK = wantListing == gotListing
|
||||
if listingOK && (expectedDirs == nil || len(dirs) == len(expectedDirs)) {
|
||||
// Put an extra sleep in if we did any retries just to make sure it really
|
||||
// is consistent (here is looking at you Amazon Drive!)
|
||||
// is consistent
|
||||
if i != 1 {
|
||||
extraSleep := 5*time.Second + sleep
|
||||
t.Logf("Sleeping for %v just to make sure", extraSleep)
|
||||
|
||||
@ -15,9 +15,6 @@ tests:
|
||||
- path: cmd/selfupdate
|
||||
localonly: true
|
||||
backends:
|
||||
# - backend: "amazonclouddrive"
|
||||
# remote: "TestAmazonCloudDrive:"
|
||||
# fastlist: false
|
||||
- backend: "local"
|
||||
remote: ""
|
||||
fastlist: false
|
||||
|
||||
@ -163,38 +163,6 @@ func TestDefaultPacer(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestAmazonCloudDrivePacer(t *testing.T) {
|
||||
c := NewAmazonCloudDrive(MinSleep(1 * time.Millisecond))
|
||||
// Do lots of times because of the random number!
|
||||
for _, test := range []struct {
|
||||
state State
|
||||
want time.Duration
|
||||
}{
|
||||
{State{SleepTime: 1 * time.Millisecond, ConsecutiveRetries: 0}, 1 * time.Millisecond},
|
||||
{State{SleepTime: 10 * time.Millisecond, ConsecutiveRetries: 0}, 1 * time.Millisecond},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 1}, 500 * time.Millisecond},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 2}, 1 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 3}, 2 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 4}, 4 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 5}, 8 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 6}, 16 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 7}, 32 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 8}, 64 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 9}, 128 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 10}, 128 * time.Second},
|
||||
{State{SleepTime: 1 * time.Second, ConsecutiveRetries: 11}, 128 * time.Second},
|
||||
} {
|
||||
const n = 1000
|
||||
var sum time.Duration
|
||||
// measure average time over n cycles
|
||||
for i := 0; i < n; i++ {
|
||||
sum += c.Calculate(test.state)
|
||||
}
|
||||
got := sum / n
|
||||
assert.False(t, got < (test.want*9)/10 || got > (test.want*11)/10, "test: %+v", test)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureIMDSPacer(t *testing.T) {
|
||||
c := NewAzureIMDS()
|
||||
for _, test := range []struct {
|
||||
|
||||
@ -113,70 +113,6 @@ func (c *ZeroDelayCalculator) Calculate(state State) time.Duration {
|
||||
return 0
|
||||
}
|
||||
|
||||
// AmazonCloudDrive is a specialized pacer for Amazon Drive
|
||||
//
|
||||
// It implements a truncated exponential backoff strategy with randomization.
|
||||
// Normally operations are paced at the interval set with SetMinSleep. On errors
|
||||
// the sleep timer is set to 0..2**retries seconds.
|
||||
//
|
||||
// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
|
||||
type AmazonCloudDrive struct {
|
||||
minSleep time.Duration // minimum sleep time
|
||||
}
|
||||
|
||||
// AmazonCloudDriveOption is the interface implemented by all options for the AmazonCloudDrive Calculator
|
||||
type AmazonCloudDriveOption interface {
|
||||
ApplyAmazonCloudDrive(*AmazonCloudDrive)
|
||||
}
|
||||
|
||||
// NewAmazonCloudDrive returns a new AmazonCloudDrive Calculator with default values
|
||||
func NewAmazonCloudDrive(opts ...AmazonCloudDriveOption) *AmazonCloudDrive {
|
||||
c := &AmazonCloudDrive{
|
||||
minSleep: 10 * time.Millisecond,
|
||||
}
|
||||
c.Update(opts...)
|
||||
return c
|
||||
}
|
||||
|
||||
// Update applies the Calculator options.
|
||||
func (c *AmazonCloudDrive) Update(opts ...AmazonCloudDriveOption) {
|
||||
for _, opt := range opts {
|
||||
opt.ApplyAmazonCloudDrive(c)
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyAmazonCloudDrive updates the value on the Calculator
|
||||
func (o MinSleep) ApplyAmazonCloudDrive(c *AmazonCloudDrive) {
|
||||
c.minSleep = time.Duration(o)
|
||||
}
|
||||
|
||||
// Calculate takes the current Pacer state and return the wait time until the next try.
|
||||
func (c *AmazonCloudDrive) Calculate(state State) time.Duration {
|
||||
if t, ok := IsRetryAfter(state.LastError); ok {
|
||||
if t < c.minSleep {
|
||||
return c.minSleep
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
consecutiveRetries := state.ConsecutiveRetries
|
||||
if consecutiveRetries == 0 {
|
||||
return c.minSleep
|
||||
}
|
||||
if consecutiveRetries > 9 {
|
||||
consecutiveRetries = 9
|
||||
}
|
||||
// consecutiveRetries starts at 1 so
|
||||
// maxSleep is 2**(consecutiveRetries-1) seconds
|
||||
maxSleep := time.Second << uint(consecutiveRetries-1)
|
||||
// actual sleep is random from 0..maxSleep
|
||||
sleepTime := time.Duration(rand.Int63n(int64(maxSleep)))
|
||||
if sleepTime < c.minSleep {
|
||||
sleepTime = c.minSleep
|
||||
}
|
||||
return sleepTime
|
||||
}
|
||||
|
||||
// AzureIMDS is a pacer for the Azure instance metadata service.
|
||||
type AzureIMDS struct {
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user