sugarsync: new backend - fixes #622

CONTRIBUTING.md

@@ -354,7 +354,7 @@ Integration tests
   * Add your backend to `fstest/test_all/config.yaml`
       * Once you've done that then you can use the integration test framework from the project root:
       * go install ./...
-      * test_all -backend remote
+      * test_all -backends remote
 
 Or if you want to run the integration tests manually:
 

README.md

@@ -62,6 +62,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)

backend/all/all.go

@@ -32,6 +32,7 @@ import (
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/sftp"
 	_ "github.com/rclone/rclone/backend/sharefile"
+	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/webdav"

backend/sugarsync/api/types.go

@@ -0,0 +1,160 @@
+// Package api has type definitions for sugarsync
+//
+// Converted from the API docs with help from https://www.onlinetool.io/xmltogo/
+package api
+
+import (
+	"encoding/xml"
+	"time"
+)
+
+// AppAuthorization is used to request a refresh token
+//
+// The token is returned in the Location: field
+type AppAuthorization struct {
+	XMLName          xml.Name `xml:"appAuthorization"`
+	Username         string   `xml:"username"`
+	Password         string   `xml:"password"`
+	Application      string   `xml:"application"`
+	AccessKeyID      string   `xml:"accessKeyId"`
+	PrivateAccessKey string   `xml:"privateAccessKey"`
+}
+
+// TokenAuthRequest is the request to get Authorization
+type TokenAuthRequest struct {
+	XMLName          xml.Name `xml:"tokenAuthRequest"`
+	AccessKeyID      string   `xml:"accessKeyId"`
+	PrivateAccessKey string   `xml:"privateAccessKey"`
+	RefreshToken     string   `xml:"refreshToken"`
+}
+
+// Authorization is returned from the TokenAuthRequest
+type Authorization struct {
+	XMLName    xml.Name  `xml:"authorization"`
+	Expiration time.Time `xml:"expiration"`
+	User       string    `xml:"user"`
+}
+
+// File represents a single file
+type File struct {
+	Name            string    `xml:"displayName"`
+	Ref             string    `xml:"ref"`
+	DsID            string    `xml:"dsid"`
+	TimeCreated     time.Time `xml:"timeCreated"`
+	Parent          string    `xml:"parent"`
+	Size            int64     `xml:"size"`
+	LastModified    time.Time `xml:"lastModified"`
+	MediaType       string    `xml:"mediaType"`
+	PresentOnServer bool      `xml:"presentOnServer"`
+	FileData        string    `xml:"fileData"`
+	Versions        string    `xml:"versions"`
+	PublicLink      PublicLink
+}
+
+// Collection represents
+// - Workspace Collection
+// - Sync Folders collection
+// - Folder
+type Collection struct {
+	Type        string    `xml:"type,attr"`
+	Name        string    `xml:"displayName"`
+	Ref         string    `xml:"ref"` // only for Folder
+	DsID        string    `xml:"dsid"`
+	TimeCreated time.Time `xml:"timeCreated"`
+	Parent      string    `xml:"parent"`
+	Collections string    `xml:"collections"`
+	Files       string    `xml:"files"`
+	Contents    string    `xml:"contents"`
+	// Sharing     bool      `xml:"sharing>enabled,attr"`
+}
+
+// CollectionContents is the result of a list call
+type CollectionContents struct {
+	//XMLName     xml.Name     `xml:"collectionContents"`
+	Start       int          `xml:"start,attr"`
+	HasMore     bool         `xml:"hasMore,attr"`
+	End         int          `xml:"end,attr"`
+	Collections []Collection `xml:"collection"`
+	Files       []File       `xml:"file"`
+}
+
+// User is returned from the /user call
+type User struct {
+	XMLName  xml.Name `xml:"user"`
+	Username string   `xml:"username"`
+	Nickname string   `xml:"nickname"`
+	Quota    struct {
+		Limit int64 `xml:"limit"`
+		Usage int64 `xml:"usage"`
+	} `xml:"quota"`
+	Workspaces            string `xml:"workspaces"`
+	SyncFolders           string `xml:"syncfolders"`
+	Deleted               string `xml:"deleted"`
+	MagicBriefcase        string `xml:"magicBriefcase"`
+	WebArchive            string `xml:"webArchive"`
+	MobilePhotos          string `xml:"mobilePhotos"`
+	Albums                string `xml:"albums"`
+	RecentActivities      string `xml:"recentActivities"`
+	ReceivedShares        string `xml:"receivedShares"`
+	PublicLinks           string `xml:"publicLinks"`
+	MaximumPublicLinkSize int    `xml:"maximumPublicLinkSize"`
+}
+
+// CreateFolder is posted to a folder URL to create a folder
+type CreateFolder struct {
+	XMLName xml.Name `xml:"folder"`
+	Name    string   `xml:"displayName"`
+}
+
+// MoveFolder is posted to a folder URL to move a folder
+type MoveFolder struct {
+	XMLName xml.Name `xml:"folder"`
+	Name    string   `xml:"displayName"`
+	Parent  string   `xml:"parent"`
+}
+
+// CreateSyncFolder is posted to the root folder URL to create a sync folder
+type CreateSyncFolder struct {
+	XMLName xml.Name `xml:"syncFolder"`
+	Name    string   `xml:"displayName"`
+}
+
+// CreateFile is posted to a folder URL to create a file
+type CreateFile struct {
+	XMLName   xml.Name `xml:"file"`
+	Name      string   `xml:"displayName"`
+	MediaType string   `xml:"mediaType"`
+}
+
+// MoveFile is posted to a file URL to create a file
+type MoveFile struct {
+	XMLName xml.Name `xml:"file"`
+	Name    string   `xml:"displayName"`
+	Parent  string   `xml:"parent"`
+}
+
+// CopyFile copies a file from source
+type CopyFile struct {
+	XMLName xml.Name `xml:"fileCopy"`
+	Source  string   `xml:"source,attr"`
+	Name    string   `xml:"displayName"`
+}
+
+// PublicLink is the URL and enabled flag for a public link
+type PublicLink struct {
+	XMLName xml.Name `xml:"publicLink"`
+	URL     string   `xml:",chardata"`
+	Enabled bool     `xml:"enabled,attr"`
+}
+
+// SetPublicLink can be used to enable the file for sharing
+type SetPublicLink struct {
+	XMLName    xml.Name `xml:"file"`
+	PublicLink PublicLink
+}
+
+// SetLastModified sets the modified time for a file
+type SetLastModified struct {
+	XMLName      xml.Name  `xml:"file"`
+	LastModified time.Time `xml:"lastModified"`
+}

backend/sugarsync/sugarsync_internal_test.go

@@ -0,0 +1,59 @@
+package sugarsync
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestErrorHandler(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		body   string
+		code   int
+		status string
+		want   string
+	}{
+		{
+			name:   "empty",
+			body:   "",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: ""`,
+		},
+		{
+			name:   "unknown",
+			body:   "<h1>unknown</h1>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: "<h1>unknown</h1>"`,
+		},
+		{
+			name:   "blank",
+			body:   "Nothing here <h3></h3>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: "Nothing here <h3></h3>"`,
+		},
+		{
+			name:   "real",
+			body:   "<h1>an error</h1>\n<h3>Can not move sync folder.</h3>\n<p>more stuff</p>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error): Can not move sync folder.`,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			resp := http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(test.body)),
+				StatusCode: test.code,
+				Status:     test.status,
+			}
+			got := errorHandler(&resp)
+			assert.Equal(t, test.want, got.Error())
+		})
+	}
+}

backend/sugarsync/sugarsync_test.go

@@ -0,0 +1,17 @@
+// Test Sugarsync filesystem interface
+package sugarsync_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/sugarsync"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSugarSync:Test",
+		NilObject:  (*sugarsync.Object)(nil),
+	})
+}

bin/make_manual.py

@@ -54,6 +54,7 @@ docs = [
     "premiumizeme.md",
     "putio.md",
     "sftp.md",
+    "sugarsync.md",
     "union.md",
     "webdav.md",
     "yandex.md",

docs/content/about.md

@@ -51,6 +51,7 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}}
 * {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}
 * {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
+* {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
 * {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
 * {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
 * {{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}

docs/content/docs.md

@@ -50,6 +50,7 @@ See the following for detailed instructions for
   * [put.io](/putio/)
   * [QingStor](/qingstor/)
   * [SFTP](/sftp/)
+  * [SugarSync](/sugarsync/)
   * [Union](/union/)
   * [WebDAV](/webdav/)
   * [Yandex Disk](/yandex/)

docs/content/overview.md

@@ -43,6 +43,7 @@ Here is an overview of the major features of each cloud storage system.
 | put.io                       | CRC-32      | Yes     | No               | Yes             | R         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
 | SFTP                         | MD5, SHA1 ‡ | Yes     | Depends          | No              | -         |
+| SugarSync                    | -           | No      | No               | No              | -         |
 | WebDAV                       | MD5, SHA1 ††| Yes ††† | Depends          | No              | -         |
 | Yandex Disk                  | MD5         | Yes     | No               | No              | R/W       |
 | The local filesystem         | All         | Yes     | Depends          | No              | -         |
@@ -340,6 +341,7 @@ operations more efficient.
 | put.io                       | Yes   | No   | Yes  | Yes     | Yes     | No    | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes |
 | QingStor                     | No    | Yes  | No   | No      | No      | Yes   | No           | No [#2178](https://github.com/rclone/rclone/issues/2178) | No  | No |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes  | Yes |
+| SugarSync                    | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | Yes         | No  | Yes |
 | WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes  | Yes |
 | Yandex Disk                  | Yes   | Yes  | Yes  | Yes     | Yes     | No    | Yes          | Yes         | Yes | Yes |
 | The local filesystem         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No          | Yes | Yes |

docs/content/sugarsync.md

@@ -0,0 +1,252 @@
+---
+title: "SugarSync"
+description: "Rclone docs for SugarSync"
+date: "2020-01-17"
+---
+
+<i class="fas fa-dove"></i> SugarSync
+-----------------------------------------
+
+[SugarSync](https://sugarsync.com) is a cloud service that enables
+active synchronization of files across computers and other devices for
+file backup, access, syncing, and sharing.
+
+The initial setup for SugarSync involves getting a token from SugarSync which you
+can do with rclone. `rclone config` walks you through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+[snip]
+XX / Sugarsync
+   \ "sugarsync"
+[snip]
+Storage> sugarsync
+** See help for sugarsync backend at: https://rclone.org/sugarsync/ **
+
+Sugarsync App ID.
+Leave blank to use rclone's.
+Enter a string value. Press Enter for the default ("").
+app_id> 
+Sugarsync Access Key ID.
+Leave blank to use rclone's.
+Enter a string value. Press Enter for the default ("").
+access_key_id> 
+Sugarsync Private Access Key
+Leave blank to use rclone's.
+Enter a string value. Press Enter for the default ("").
+private_access_key> 
+Permanently delete files if true
+otherwise put them in the deleted files.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+hard_delete> 
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+Username (email address)> nick@craig-wood.com
+Your Sugarsync password is only required during setup and will not be stored.
+password:
+--------------------
+[remote]
+type = sugarsync
+refresh_token = https://api.sugarsync.com/app-authorization/XXXXXXXXXXXXXXXXXX
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Note that the config asks for your email and password but doesn't
+store them, it only uses them to get the initial token.
+
+Once configured you can then use `rclone` like this,
+
+List directories (sync folders) in top level of your SugarSync
+
+    rclone lsd remote:
+
+List all the files in your SugarSync folder "Test"
+
+    rclone ls remote:Test
+
+To copy a local directory to an SugarSync folder called backup
+
+    rclone copy /home/source remote:backup
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, eg `remote:directory/subdirectory`.
+
+**NB** you can't create files in the top level folder you have to
+create a folder, which rclone will create as a "Sync Folder" with
+SugarSync.
+
+### Modified time and hashes ###
+
+SugarSync does not support modification times or hashes, therefore
+syncing will default to `--size-only` checking.  Note that using
+`--update` will work as rclone can read the time files were uploaded.
+
+#### Restricted filename characters
+
+SugarSync replaces the [default restricted characters set](/overview/#restricted-characters)
+except for DEL.
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
+as they can't be used in XML strings.
+
+### Deleting files ###
+
+Deleted files will be moved to the "Deleted items" folder by default.
+
+However you can supply the flag `--sugarsync-hard-delete` or set the
+config parameter `hard_delete = true` if you would like files to be
+deleted straight away.
+
+
+<!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/sugarsync/sugarsync.go then run make backenddocs -->
+### Standard Options
+
+Here are the standard options specific to sugarsync (Sugarsync).
+
+#### --sugarsync-app-id
+
+Sugarsync App ID.
+
+Leave blank to use rclone's.
+
+- Config:      app_id
+- Env Var:     RCLONE_SUGARSYNC_APP_ID
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-access-key-id
+
+Sugarsync Access Key ID.
+
+Leave blank to use rclone's.
+
+- Config:      access_key_id
+- Env Var:     RCLONE_SUGARSYNC_ACCESS_KEY_ID
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-private-access-key
+
+Sugarsync Private Access Key
+
+Leave blank to use rclone's.
+
+- Config:      private_access_key
+- Env Var:     RCLONE_SUGARSYNC_PRIVATE_ACCESS_KEY
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-hard-delete
+
+Permanently delete files if true
+otherwise put them in the deleted files.
+
+- Config:      hard_delete
+- Env Var:     RCLONE_SUGARSYNC_HARD_DELETE
+- Type:        bool
+- Default:     false
+
+### Advanced Options
+
+Here are the advanced options specific to sugarsync (Sugarsync).
+
+#### --sugarsync-refresh-token
+
+Sugarsync refresh token
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      refresh_token
+- Env Var:     RCLONE_SUGARSYNC_REFRESH_TOKEN
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-authorization
+
+Sugarsync authorization
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      authorization
+- Env Var:     RCLONE_SUGARSYNC_AUTHORIZATION
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-authorization-expiry
+
+Sugarsync authorization expiry
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      authorization_expiry
+- Env Var:     RCLONE_SUGARSYNC_AUTHORIZATION_EXPIRY
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-user
+
+Sugarsync user
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      user
+- Env Var:     RCLONE_SUGARSYNC_USER
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-root-id
+
+Sugarsync root id
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      root_id
+- Env Var:     RCLONE_SUGARSYNC_ROOT_ID
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-deleted-id
+
+Sugarsync deleted folder id
+
+Leave blank normally, will be auto configured by rclone.
+
+- Config:      deleted_id
+- Env Var:     RCLONE_SUGARSYNC_DELETED_ID
+- Type:        string
+- Default:     ""
+
+#### --sugarsync-encoding
+
+This sets the encoding for the backend.
+
+See: the [encoding section in the overview](/overview/#encoding) for more info.
+
+- Config:      encoding
+- Env Var:     RCLONE_SUGARSYNC_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,Ctl,InvalidUtf8,Dot
+
+<!--- autogenerated options stop -->

docs/layouts/chrome/navbar.html

@@ -86,6 +86,7 @@
                     <li><a href="/premiumizeme/"><i class="fa fa-user"></i> premiumize.me</a></li>
                     <li><a href="/putio/"><i class="fas fa-parking"></i> put.io</a></li>
                     <li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li>
+                    <li><a href="/sugarsync/"><i class="fas fa-dove"></i> SugarSync</a></li>
                     <li><a href="/union/"><i class="fa fa-link"></i> Union (merge backends)</a></li>
                     <li><a href="/webdav/"><i class="fa fa-server"></i> WebDAV</a></li>
                     <li><a href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a></li>

fstest/test_all/config.yaml

@@ -143,6 +143,11 @@ backends:
  - backend:  "sftp"
    remote:   "TestSftp:"
    fastlist: false
+ - backend:  "sugarsync"
+   remote:   "TestSugarSync:Test"
+   fastlist: false
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/PublicLink
  - backend:  "swift"
    remote:   "TestSwift:"
    fastlist: true