quatrix: add backend to support Quatrix

Co-authored-by: Volodymyr Kit <v.kit@maytech.net>

README.md

@@ -75,6 +75,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
+  * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)

backend/all/all.go

@@ -41,6 +41,7 @@ import (
 	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
+	_ "github.com/rclone/rclone/backend/quatrix"
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/seafile"
 	_ "github.com/rclone/rclone/backend/sftp"

backend/quatrix/api/types.go

@@ -0,0 +1,182 @@
+// Package api provides types used by the Quatrix API.
+package api
+
+import (
+	"strconv"
+	"time"
+)
+
+// OverwriteOnCopyMode is a conflict resolve mode during copy. Files with conflicting names will be overwritten
+const OverwriteOnCopyMode = "overwrite"
+
+// ProfileInfo is a profile info about quota
+type ProfileInfo struct {
+	UserUsed  int64 `json:"user_used"`
+	UserLimit int64 `json:"user_limit"`
+	AccUsed   int64 `json:"acc_used"`
+	AccLimit  int64 `json:"acc_limit"`
+}
+
+// IDList is a general object that contains list of ids
+type IDList struct {
+	IDs []string `json:"ids"`
+}
+
+// DeleteParams is the request to delete object
+type DeleteParams struct {
+	IDs               []string `json:"ids"`
+	DeletePermanently bool     `json:"delete_permanently"`
+}
+
+// FileInfoParams is the request to get object's (file or directory) info
+type FileInfoParams struct {
+	ParentID string `json:"parent_id,omitempty"`
+	Path     string `json:"path"`
+}
+
+// FileInfo is the response to get object's (file or directory) info
+type FileInfo struct {
+	FileID   string `json:"file_id"`
+	ParentID string `json:"parent_id"`
+	Src      string `json:"src"`
+	Type     string `json:"type"`
+}
+
+// IsFile returns true if object is a file
+// false otherwise
+func (fi *FileInfo) IsFile() bool {
+	if fi == nil {
+		return false
+	}
+
+	return fi.Type == "F"
+}
+
+// IsDir returns true if object is a directory
+// false otherwise
+func (fi *FileInfo) IsDir() bool {
+	if fi == nil {
+		return false
+	}
+
+	return fi.Type == "D" || fi.Type == "S" || fi.Type == "T"
+}
+
+// CreateDirParams is the request to create a directory
+type CreateDirParams struct {
+	Target  string `json:"target,omitempty"`
+	Name    string `json:"name"`
+	Resolve bool   `json:"resolve"`
+}
+
+// File represent metadata about object in Quatrix (file or directory)
+type File struct {
+	ID         string   `json:"id"`
+	Created    JSONTime `json:"created"`
+	Modified   JSONTime `json:"modified"`
+	Name       string   `json:"name"`
+	ParentID   string   `json:"parent_id"`
+	Size       int64    `json:"size"`
+	ModifiedMS JSONTime `json:"modified_ms"`
+	Type       string   `json:"type"`
+	Operations int      `json:"operations"`
+	SubType    string   `json:"sub_type"`
+	Content    []File   `json:"content"`
+}
+
+// IsFile returns true if object is a file
+// false otherwise
+func (f *File) IsFile() bool {
+	if f == nil {
+		return false
+	}
+
+	return f.Type == "F"
+}
+
+// IsDir returns true if object is a directory
+// false otherwise
+func (f *File) IsDir() bool {
+	if f == nil {
+		return false
+	}
+
+	return f.Type == "D" || f.Type == "S" || f.Type == "T"
+}
+
+// SetMTimeParams is the request to set modification time for object
+type SetMTimeParams struct {
+	ID    string   `json:"id,omitempty"`
+	MTime JSONTime `json:"mtime"`
+}
+
+// JSONTime provides methods to marshal/unmarshal time.Time as Unix time
+type JSONTime time.Time
+
+// MarshalJSON returns time representation in Unix time
+func (u JSONTime) MarshalJSON() ([]byte, error) {
+	return []byte(strconv.FormatFloat(float64(time.Time(u).UTC().UnixNano())/1e9, 'f', 6, 64)), nil
+}
+
+// UnmarshalJSON sets time from Unix time representation
+func (u *JSONTime) UnmarshalJSON(data []byte) error {
+	f, err := strconv.ParseFloat(string(data), 64)
+	if err != nil {
+		return err
+	}
+
+	t := JSONTime(time.Unix(0, int64(f*1e9)))
+	*u = t
+
+	return nil
+}
+
+// String returns Unix time representation of time as string
+func (u JSONTime) String() string {
+	return strconv.FormatInt(time.Time(u).UTC().Unix(), 10)
+}
+
+// DownloadLinkResponse is the response to download-link request
+type DownloadLinkResponse struct {
+	ID string `json:"id"`
+}
+
+// UploadLinkParams is the request to get upload-link
+type UploadLinkParams struct {
+	Name     string `json:"name"`
+	ParentID string `json:"parent_id"`
+	Resolve  bool   `json:"resolve"`
+}
+
+// UploadLinkResponse is the response to upload-link request
+type UploadLinkResponse struct {
+	Name      string `json:"name"`
+	FileID    string `json:"file_id"`
+	ParentID  string `json:"parent_id"`
+	UploadKey string `json:"upload_key"`
+}
+
+// UploadFinalizeResponse is the response to finalize file method
+type UploadFinalizeResponse struct {
+	FileID   string `json:"id"`
+	ParentID string `json:"parent_id"`
+	Modified int64  `json:"modified"`
+	FileSize int64  `json:"size"`
+}
+
+// FileModifyParams is the request to get modify file link
+type FileModifyParams struct {
+	ID       string `json:"id"`
+	Truncate int64  `json:"truncate"`
+}
+
+// FileCopyMoveOneParams is the request to do server-side copy and move
+// can be used for file or directory
+type FileCopyMoveOneParams struct {
+	ID          string   `json:"file_id"`
+	Target      string   `json:"target_id"`
+	Name        string   `json:"name"`
+	MTime       JSONTime `json:"mtime"`
+	Resolve     bool     `json:"resolve"`
+	ResolveMode string   `json:"resolve_mode"`
+}

backend/quatrix/quatrix_test.go

@@ -0,0 +1,17 @@
+// Test Quatrix filesystem interface
+package quatrix_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/quatrix"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestQuatrix:",
+		NilObject:  (*quatrix.Object)(nil),
+	})
+}

backend/quatrix/upload_memory.go

@@ -0,0 +1,108 @@
+package quatrix
+
+import (
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// UploadMemoryManager dynamically calculates every chunk size for the transfer and increases or decreases it
+// depending on the upload speed. This makes general upload time smaller, because transfers that are faster
+// does not have to wait for the slower ones until they finish upload.
+type UploadMemoryManager struct {
+	m              sync.Mutex
+	useDynamicSize bool
+	shared         int64
+	reserved       int64
+	effectiveTime  time.Duration
+	fileUsage      map[string]int64
+}
+
+// NewUploadMemoryManager is a constructor for UploadMemoryManager
+func NewUploadMemoryManager(ci *fs.ConfigInfo, opt *Options) *UploadMemoryManager {
+	useDynamicSize := true
+
+	sharedMemory := int64(opt.MaximalSummaryChunkSize) - int64(opt.MinimalChunkSize)*int64(ci.Transfers)
+	if sharedMemory <= 0 {
+		sharedMemory = 0
+		useDynamicSize = false
+	}
+
+	return &UploadMemoryManager{
+		useDynamicSize: useDynamicSize,
+		shared:         sharedMemory,
+		reserved:       int64(opt.MinimalChunkSize),
+		effectiveTime:  time.Duration(opt.EffectiveUploadTime),
+		fileUsage:      map[string]int64{},
+	}
+}
+
+// Consume -- decide amount of memory to consume
+func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed float64) int64 {
+	if !u.useDynamicSize {
+		if neededMemory < u.reserved {
+			return neededMemory
+		}
+
+		return u.reserved
+	}
+
+	u.m.Lock()
+	defer u.m.Unlock()
+
+	borrowed, found := u.fileUsage[fileID]
+	if found {
+		u.shared += borrowed
+		borrowed = 0
+	}
+
+	defer func() { u.fileUsage[fileID] = borrowed }()
+
+	effectiveChunkSize := int64(speed * u.effectiveTime.Seconds())
+
+	if effectiveChunkSize < u.reserved {
+		effectiveChunkSize = u.reserved
+	}
+
+	if neededMemory < effectiveChunkSize {
+		effectiveChunkSize = neededMemory
+	}
+
+	if effectiveChunkSize <= u.reserved {
+		return effectiveChunkSize
+	}
+
+	toBorrow := effectiveChunkSize - u.reserved
+
+	if toBorrow <= u.shared {
+		u.shared -= toBorrow
+		borrowed = toBorrow
+
+		return effectiveChunkSize
+	}
+
+	borrowed = u.shared
+	u.shared = 0
+
+	return borrowed + u.reserved
+}
+
+// Return returns consumed memory for the previous chunk upload to the memory pool
+func (u *UploadMemoryManager) Return(fileID string) {
+	if !u.useDynamicSize {
+		return
+	}
+
+	u.m.Lock()
+	defer u.m.Unlock()
+
+	borrowed, found := u.fileUsage[fileID]
+	if !found {
+		return
+	}
+
+	u.shared += borrowed
+
+	delete(u.fileUsage, fileID)
+}

bin/make_manual.py

@@ -61,6 +61,7 @@ docs = [
     "opendrive.md",
     "oracleobjectstorage.md",
     "qingstor.md",
+    "quatrix.md",
     "sia.md",
     "swift.md",
     "pcloud.md",

docs/content/_index.md

@@ -160,6 +160,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="put.io" home="https://put.io/" config="/putio/" >}}
 {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
 {{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}}
+{{< provider name="Quatrix by Maytech" home="https://www.maytech.net/products/quatrix-business" config="/quatrix/" >}}
 {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
 {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}}
 {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}

docs/content/docs.md

@@ -67,6 +67,7 @@ See the following for detailed instructions for
   * [premiumize.me](/premiumizeme/)
   * [put.io](/putio/)
   * [QingStor](/qingstor/)
+  * [Quatrix by Maytech](/quatrix/)
   * [Seafile](/seafile/)
   * [SFTP](/sftp/)
   * [Sia](/sia/)

docs/content/overview.md

@@ -48,6 +48,7 @@ Here is an overview of the major features of each cloud storage system.
 | premiumize.me                | -                | -       | Yes              | No              | R         | -        |
 | put.io                       | CRC-32           | R/W     | No               | Yes             | R         | -        |
 | QingStor                     | MD5              | - ⁹     | No               | No              | R/W       | -        |
+| Quatrix by Maytech           | -                | R/W     | No               | No              | -         | -        |
 | Seafile                      | -                | -       | No               | No              | -         | -        |
 | SFTP                         | MD5, SHA1 ²      | R/W     | Depends          | No              | -         | -        |
 | Sia                          | -                | -       | No               | No              | -         | -        |
@@ -499,6 +500,7 @@ upon backend-specific capabilities.
 | premiumize.me                | Yes   | No   | Yes  | Yes     | No      | No    | No           | No  | Yes          | Yes   | Yes      |
 | put.io                       | Yes   | No   | Yes  | Yes     | Yes     | No    | Yes          | No  | No           | Yes   | Yes      |
 | QingStor                     | No    | Yes  | No   | No      | Yes     | Yes   | No           | No  | No           | No    | No       |
+| Quatrix by Maytech           | Yes   | Yes  | Yes  | Yes     | No      | No    | No           | No  | No           | Yes   | Yes      |
 | Seafile                      | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | No  | Yes          | Yes   | Yes      |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          | No  | No           | Yes   | Yes      |
 | Sia                          | No    | No   | No   | No      | No      | No    | Yes          | No  | No           | No    | Yes      |

docs/content/quatrix.md

@@ -0,0 +1,249 @@
+---
+title: "Quatrix"
+description: "Rclone docs for Quatrix"
+versionIntroduced: "v1.63.2"
+---
+
+# {{< icon "fas fa-shield-alt" >}} Quatrix
+
+Quatrix by Maytech is [Quatrix Secure Compliant File Sharing | Maytech](https://www.maytech.net/products/quatrix-business).
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, e.g., `remote:directory/subdirectory`.
+
+The initial setup for Quatrix involves getting an API Key from Quatrix. You can get the API key in the user's profile at `https://<account>/profile/api-keys`
+or with the help of the API - https://docs.maytech.net/quatrix/quatrix-api/api-explorer#/API-Key/post_api_key_create.
+
+See complete Swagger documentation for Quatrix - https://docs.maytech.net/quatrix/quatrix-api/api-explorer
+
+## Configuration
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Quatrix by Maytech
+   \ "quatrix"
+[snip]
+Storage> quatrix
+API key for accessing Quatrix account.
+api_key> your_api_key
+Host name of Quatrix account.
+host> example.quatrix.it
+
+--------------------
+[remote]
+api_key = your_api_key
+host = example.quatrix.it
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your Quatrix
+
+    rclone lsd remote:
+
+List all the files in your Quatrix
+
+    rclone ls remote:
+
+To copy a local directory to an Quatrix directory called backup
+
+    rclone copy /home/source remote:backup
+
+### API key validity
+
+API Key is created with no expiration date. It will be valid until you delete or deactivate it in your account.
+After disabling, the API Key can be enabled back. If the API Key was deleted and a new key was created, you can
+update it in rclone config. The same happens if the hostname was changed.
+
+```
+$ rclone config
+Current remotes:
+
+Name                 Type
+====                 ====
+remote               quatrix
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> e
+Choose a number from below, or type in an existing value
+ 1 > remote
+remote> remote
+--------------------
+[remote]
+type = quatrix
+host = some_host.quatrix.it
+api_key = your_api_key
+--------------------
+Edit remote
+Option api_key.
+API key for accessing Quatrix account
+Enter a string value. Press Enter for the default (your_api_key)
+api_key>
+Option host.
+Host name of Quatrix account
+Enter a string value. Press Enter for the default (some_host.quatrix.it).
+
+--------------------
+[remote]
+type = quatrix
+host = some_host.quatrix.it
+api_key = your_api_key
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+### Modified time and hashes
+
+Quatrix allows modification times to be set on objects accurate to 1 microsecond.
+These will be used to detect whether objects need syncing or not.
+
+Quatrix does not support hashes, so you cannot use the `--checksum` flag.
+
+### Restricted filename characters
+
+File names in Quatrix are case sensitive and have limitations like the maximum length of a filename is 255, and the minimum length is 1. A file name cannot be equal to `.` or `..` nor contain `/` , `\` or non-printable ascii.
+
+### Transfers
+
+For files above 50 MiB rclone will use a chunked transfer. Rclone will upload up to `--transfers` chunks at the same time (shared among all multipart uploads).
+Chunks are buffered in memory, and the minimal chunk size is 10_000_000 bytes by default, and it can be changed in the advanced configuration, so increasing `--transfers` will increase the memory use.
+The chunk size has a maximum size limit, which is set to 100_000_000 bytes by default and can be changed in the advanced configuration.
+The size of the uploaded chunk will dynamically change depending on the upload speed.
+The total memory use equals the number of transfers multiplied by the minimal chunk size.
+In case there's free memory allocated for the upload (which equals the difference of `maximal_summary_chunk_size` and `minimal_chunk_size` * `transfers`),
+the chunk size may increase in case of high upload speed. As well as it can decrease in case of upload speed problems.
+If no free memory is available, all chunks will equal `minimal_chunk_size`.
+
+### Deleting files
+
+Files you delete with rclone will end up in Trash and be stored there for 30 days.
+Quatrix also provides an API to permanently delete files and an API to empty the Trash so that you can remove files permanently from your account.
+
+{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/quatrix/quatrix.go then run make backenddocs" >}}
+### Standard options
+
+Here are the Standard options specific to quatrix (Quatrix by Maytech).
+
+#### --quatrix-api-key
+
+API key for accessing Quatrix account
+
+Properties:
+
+- Config:      api_key
+- Env Var:     RCLONE_QUATRIX_API_KEY
+- Type:        string
+- Required:    true
+
+#### --quatrix-host
+
+Host name of Quatrix account
+
+Properties:
+
+- Config:      host
+- Env Var:     RCLONE_QUATRIX_HOST
+- Type:        string
+- Required:    true
+
+### Advanced options
+
+Here are the Advanced options specific to quatrix (Quatrix by Maytech).
+
+#### --quatrix-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config:      encoding
+- Env Var:     RCLONE_QUATRIX_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
+
+#### --quatrix-effective-upload-time
+
+Wanted upload time for one chunk
+
+Properties:
+
+- Config:      effective_upload_time
+- Env Var:     RCLONE_QUATRIX_EFFECTIVE_UPLOAD_TIME
+- Type:        string
+- Default:     "4s"
+
+#### --quatrix-minimal-chunk-size
+
+The minimal size for one chunk
+
+Properties:
+
+- Config:      minimal_chunk_size
+- Env Var:     RCLONE_QUATRIX_MINIMAL_CHUNK_SIZE
+- Type:        SizeSuffix
+- Default:     9.537Mi
+
+#### --quatrix-maximal-summary-chunk-size
+
+The maximal summary for all chunks. It should not be less than 'transfers'*'minimal_chunk_size'
+
+Properties:
+
+- Config:      maximal_summary_chunk_size
+- Env Var:     RCLONE_QUATRIX_MAXIMAL_SUMMARY_CHUNK_SIZE
+- Type:        SizeSuffix
+- Default:     95.367Mi
+
+#### --quatrix-hard-delete
+
+Delete files permanently rather than putting them into the trash.
+
+Properties:
+
+- Config:      hard_delete
+- Env Var:     RCLONE_QUATRIX_HARD_DELETE
+- Type:        bool
+- Default:     false
+
+{{< rem autogenerated options stop >}}
+
+## Storage usage
+
+The storage usage in Quatrix is restricted to the account during the purchase. You can restrict any user with a smaller storage limit.
+The account limit is applied if the user has no custom storage limit. Once you've reached the limit, the upload of files will fail.
+This can be fixed by freeing up the space or increasing the quota.
+
+## Server-side operations
+
+Quatrix supports server-side operations (copy and move). In case of conflict, files are overwritten during server-side operation.

docs/layouts/chrome/navbar.html

@@ -90,6 +90,7 @@
           <a class="dropdown-item" href="/pikpak/"><i class="fa fa-cloud fa-fw"></i> PikPak</a>
           <a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user fa-fw"></i> premiumize.me</a>
           <a class="dropdown-item" href="/putio/"><i class="fas fa-parking fa-fw"></i> put.io</a>
+          <a class="dropdown-item" href="/quatrix/"><i class="fas fa-shield-alt"></i> Quatrix</a>
           <a class="dropdown-item" href="/seafile/"><i class="fa fa-server fa-fw"></i> Seafile</a>
           <a class="dropdown-item" href="/sftp/"><i class="fa fa-server fa-fw"></i> SFTP</a>
           <a class="dropdown-item" href="/sia/"><i class="fa fa-globe fa-fw"></i> Sia</a>

fstest/test_all/config.yaml

@@ -434,3 +434,6 @@ backends:
      - TestIntegration/FsMkdir/FsEncoding/trailing_LF
      - TestIntegration/FsMkdir/FsEncoding/leading_HT
      - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0
+ - backend:  "quatrix"
+   remote:   "TestQuatrix:"
+   fastlist: false