From 1f846c18d432460de598bdee80e2ab6434c0e31e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 8 Jun 2021 01:59:57 -0700 Subject: [PATCH] s3: Add SeaweedFS --- README.md | 1 + backend/s3/s3.go | 9 ++++++- docs/content/_index.md | 1 + docs/content/s3.md | 55 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4801a0805..115f37161 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/) * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway) * Seafile [:page_facing_up:](https://rclone.org/seafile/) + * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs) * SFTP [:page_facing_up:](https://rclone.org/sftp/) * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath) * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index ccdeebf14..ac3d99eb5 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -58,7 +58,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, SeaweedFS, and Tencent COS", NewFs: NewFs, CommandHelp: commandHelp, Options: []fs.Option{{ @@ -91,6 +91,9 @@ func init() { }, { Value: "Scaleway", Help: "Scaleway Object Storage", + }, { + Value: "SeaweedFS", + Help: "SeaweedFS S3", }, { Value: "StackPath", Help: "StackPath Object Storage", @@ -592,6 +595,10 @@ func init() { Value: "sgp1.digitaloceanspaces.com", Help: "Digital Ocean Spaces Singapore 1", Provider: "DigitalOcean", + }, { + Value: "localhost:8333", + Help: "SeaweedFS S3 localhost", + Provider: "SeaweedFS", }, { Value: "s3.wasabisys.com", Help: "Wasabi US East endpoint", diff --git a/docs/content/_index.md b/docs/content/_index.md index 2a97ae6c8..e1e72cde1 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -147,6 +147,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}} {{< provider name="Seafile" home="https://www.seafile.com/" config="/seafile/" >}} +{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index b8d2a1781..c30049663 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -17,6 +17,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} +{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} {{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}} {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}} @@ -549,7 +550,7 @@ Vault API, so rclone cannot directly access Glacier Vaults. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}} ### Standard Options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS). +Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, SeaweedFS, and Tencent COS). #### --s3-provider @@ -578,6 +579,8 @@ Choose your S3 provider. - Netease Object Storage (NOS) - "Scaleway" - Scaleway Object Storage + - "SeaweedFS" + - SeaweedFS S3 - "StackPath" - StackPath Object Storage - "TencentCOS" @@ -2268,6 +2271,56 @@ server_side_encryption = storage_class = ``` +### SeaweedFS ### + +[SeaweedFS](https://github.com/chrislusf/seaweedfs/) is a distributed storage system for +blobs, objects, files, and data lake, with O(1) disk seek and a scalable file metadata store. +It has an S3 compatible object storage interface. + +Assuming the SeaweedFS are configured with `weed shell` as such: +``` +> s3.bucket.create -name foo +> s3.configure -access_key=any -secret_key=any -buckets=foo -user=me -actions=Read,Write,List,Tagging,Admin -apply +{ + "identities": [ + { + "name": "me", + "credentials": [ + { + "accessKey": "any", + "secretKey": "any" + } + ], + "actions": [ + "Read:foo", + "Write:foo", + "List:foo", + "Tagging:foo", + "Admin:foo" + ] + } + ] +} +``` + +To use rclone with SeaweedFS, above configuration should end up with something like this in +your config: + +``` +[seaweedfs_s3] +type = s3 +provider = SeaweedFS +access_key_id = any +secret_access_key = any +endpoint = localhost:8333 +``` + +So once set up, for example to copy files into a bucket + +``` +rclone copy /path/to/files seaweedfs_s3:foo +``` + ### Wasabi ### [Wasabi](https://wasabi.com) is a cloud-based object storage service for a