Implement Yandex storage backend - fixes #234

fs/operations_test.go

@@ -30,6 +30,7 @@ import (
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"
 	_ "github.com/ncw/rclone/swift"
+	_ "github.com/ncw/rclone/yandex"
 )
 
 // Globals

fs/test_all.go

@@ -26,6 +26,7 @@ var (
 		"TestOneDrive:",
 		"TestHubic:",
 		"TestB2:",
+		"TestYandex:",
 	}
 	binary = "fs.test"
 	// Flags

fstest/fstests/gen_tests.go

@@ -134,5 +134,6 @@ func main() {
 	generateTestProgram(t, fns, "OneDrive")
 	generateTestProgram(t, fns, "Hubic")
 	generateTestProgram(t, fns, "B2")
+	generateTestProgram(t, fns, "Yandex")
 	log.Printf("Done")
 }

rclone.go

@@ -26,6 +26,7 @@ import (
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"
 	_ "github.com/ncw/rclone/swift"
+	_ "github.com/ncw/rclone/yandex"
 )
 
 // Globals

yandex/api/api_request.go

@@ -0,0 +1,5 @@
+package src
+
+type apiRequest interface {
+	Request() *HTTPRequest
+}

yandex/api/api_upload.go

@@ -0,0 +1,34 @@
+package src
+
+//from  yadisk
+
+import (
+	"io"
+	"net/http"
+)
+
+//RootAddr is the base URL for Yandex Disk API.
+const RootAddr = "https://cloud-api.yandex.com" //also https://cloud-api.yandex.net and https://cloud-api.yandex.ru
+
+func (c *Client) setRequestScope(req *http.Request) {
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Authorization", "OAuth "+c.token)
+}
+
+func (c *Client) scopedRequest(method, urlPath string, body io.Reader) (*http.Request, error) {
+	fullURL := RootAddr
+	if urlPath[:1] != "/" {
+		fullURL += "/" + urlPath
+	} else {
+		fullURL += urlPath
+	}
+
+	req, err := http.NewRequest(method, fullURL, body)
+	if err != nil {
+		return req, err
+	}
+
+	c.setRequestScope(req)
+	return req, nil
+}

yandex/api/client.go

@@ -0,0 +1,134 @@
+package src
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+//Client struct
+type Client struct {
+	token      string
+	basePath   string
+	HTTPClient *http.Client
+}
+
+//NewClient creates new client
+func NewClient(token string, client ...*http.Client) *Client {
+	return newClientInternal(
+		token,
+		"https://cloud-api.yandex.com/v1/disk", //also "https://cloud-api.yandex.net/v1/disk" "https://cloud-api.yandex.ru/v1/disk"
+		client...)
+}
+
+func newClientInternal(token string, basePath string, client ...*http.Client) *Client {
+	c := &Client{
+		token:    token,
+		basePath: basePath,
+	}
+	if len(client) != 0 {
+		c.HTTPClient = client[0]
+	} else {
+		c.HTTPClient = http.DefaultClient
+	}
+	return c
+}
+
+//ErrorHandler type
+type ErrorHandler func(*http.Response) error
+
+var defaultErrorHandler ErrorHandler = func(resp *http.Response) error {
+	if resp.StatusCode/100 == 5 {
+		return errors.New("server error")
+	}
+
+	if resp.StatusCode/100 == 4 {
+		var response DiskClientError
+		contents, _ := ioutil.ReadAll(resp.Body)
+		err := json.Unmarshal(contents, &response)
+		if err != nil {
+			return err
+		}
+		return response
+	}
+
+	if resp.StatusCode/100 == 3 {
+		return errors.New("redirect error")
+	}
+	return nil
+}
+
+func (HTTPRequest *HTTPRequest) run(client *Client) ([]byte, error) {
+	var err error
+	values := make(url.Values)
+	if HTTPRequest.Parameters != nil {
+		for k, v := range HTTPRequest.Parameters {
+			values.Set(k, fmt.Sprintf("%v", v))
+		}
+	}
+
+	var req *http.Request
+	if HTTPRequest.Method == "POST" {
+		// TODO json serialize
+		req, err = http.NewRequest(
+			"POST",
+			client.basePath+HTTPRequest.Path,
+			strings.NewReader(values.Encode()))
+		if err != nil {
+			return nil, err
+		}
+		// TODO
+		// req.Header.Set("Content-Type", "application/json")
+	} else {
+		req, err = http.NewRequest(
+			HTTPRequest.Method,
+			client.basePath+HTTPRequest.Path+"?"+values.Encode(),
+			nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	for headerName := range HTTPRequest.Headers {
+		var headerValues = HTTPRequest.Headers[headerName]
+		for _, headerValue := range headerValues {
+			req.Header.Set(headerName, headerValue)
+		}
+	}
+	return runRequest(client, req)
+}
+
+func runRequest(client *Client, req *http.Request) ([]byte, error) {
+	return runRequestWithErrorHandler(client, req, defaultErrorHandler)
+}
+
+func runRequestWithErrorHandler(client *Client, req *http.Request, errorHandler ErrorHandler) ([]byte, error) {
+	resp, err := client.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer CheckClose(resp.Body, &err)
+
+	return checkResponseForErrorsWithErrorHandler(resp, errorHandler)
+}
+
+func checkResponseForErrorsWithErrorHandler(resp *http.Response, errorHandler ErrorHandler) ([]byte, error) {
+	if resp.StatusCode/100 > 2 {
+		return nil, errorHandler(resp)
+	}
+	return ioutil.ReadAll(resp.Body)
+}
+
+// CheckClose is a utility function used to check the return from
+// Close in a defer statement.
+func CheckClose(c io.Closer, err *error) {
+	cerr := c.Close()
+	if *err == nil {
+		*err = cerr
+	}
+}

yandex/api/custom_property.go

@@ -0,0 +1,50 @@
+package src
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/url"
+)
+
+//CustomPropertyResponse struct we send and is returned by the API for CustomProperty request.
+type CustomPropertyResponse struct {
+	CustomProperties map[string]interface{} `json:"custom_properties"`
+}
+
+//SetCustomProperty will set specified data from Yandex Disk
+func (c *Client) SetCustomProperty(remotePath string, property string, value string) error {
+	rcm := map[string]interface{}{
+		property: value,
+	}
+	cpr := CustomPropertyResponse{rcm}
+	data, _ := json.Marshal(cpr)
+	body := bytes.NewReader(data)
+	err := c.SetCustomPropertyRequest(remotePath, body)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//SetCustomPropertyRequest will make an CustomProperty request and return a URL to CustomProperty data to.
+func (c *Client) SetCustomPropertyRequest(remotePath string, body io.Reader) error {
+	values := url.Values{}
+	values.Add("path", remotePath)
+	req, err := c.scopedRequest("PATCH", "/v1/disk/resources?"+values.Encode(), body)
+	if err != nil {
+		return err
+	}
+
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	if err := CheckAPIError(resp); err != nil {
+		return err
+	}
+
+	//If needed we can read response and check if custom_property is set.
+
+	return nil
+}

yandex/api/delete.go

@@ -0,0 +1,26 @@
+package src
+
+import (
+	"net/url"
+	"strconv"
+)
+
+// Delete will remove specified file/folder from Yandex Disk
+func (c *Client) Delete(remotePath string, permanently bool) error {
+
+	values := url.Values{}
+	values.Add("permanently", strconv.FormatBool(permanently))
+	values.Add("path", remotePath)
+	urlPath := "/v1/disk/resources?" + values.Encode()
+	fullURL := RootAddr
+	if urlPath[:1] != "/" {
+		fullURL += "/" + urlPath
+	} else {
+		fullURL += urlPath
+	}
+
+	if err := c.PerformDelete(fullURL); err != nil {
+		return err
+	}
+	return nil
+}

yandex/api/disk_info_request.go

@@ -0,0 +1,48 @@
+package src
+
+import "encoding/json"
+
+//DiskInfoRequest type
+type DiskInfoRequest struct {
+	client      *Client
+	HTTPRequest *HTTPRequest
+}
+
+func (req *DiskInfoRequest) request() *HTTPRequest {
+	return req.HTTPRequest
+}
+
+//DiskInfoResponse struct is returned by the API for DiskInfo request.
+type DiskInfoResponse struct {
+	TrashSize     uint64            `json:"TrashSize"`
+	TotalSpace    uint64            `json:"TotalSpace"`
+	UsedSpace     uint64            `json:"UsedSpace"`
+	SystemFolders map[string]string `json:"SystemFolders"`
+}
+
+//NewDiskInfoRequest create new DiskInfo Request
+func (c *Client) NewDiskInfoRequest() *DiskInfoRequest {
+	return &DiskInfoRequest{
+		client:      c,
+		HTTPRequest: createGetRequest(c, "/", nil),
+	}
+}
+
+//Exec run DiskInfo Request
+func (req *DiskInfoRequest) Exec() (*DiskInfoResponse, error) {
+	data, err := req.request().run(req.client)
+	if err != nil {
+		return nil, err
+	}
+
+	var info DiskInfoResponse
+	err = json.Unmarshal(data, &info)
+	if err != nil {
+		return nil, err
+	}
+	if info.SystemFolders == nil {
+		info.SystemFolders = make(map[string]string)
+	}
+
+	return &info, nil
+}

yandex/api/download.go

@@ -0,0 +1,66 @@
+package src
+
+import (
+	"encoding/json"
+	"io"
+	"net/url"
+)
+
+// DownloadResponse struct is returned by the API for Download request.
+type DownloadResponse struct {
+	HRef      string `json:"href"`
+	Method    string `json:"method"`
+	Templated bool   `json:"templated"`
+}
+
+// Download will get specified data from Yandex.Disk.
+func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer
+	ur, err := c.DownloadRequest(remotePath)
+	if err != nil {
+		return nil, err
+	}
+	return c.PerformDownload(ur.HRef)
+}
+
+// DownloadRequest will make an download request and return a URL to download data to.
+func (c *Client) DownloadRequest(remotePath string) (*DownloadResponse, error) {
+	values := url.Values{}
+	values.Add("path", remotePath)
+
+	req, err := c.scopedRequest("GET", "/v1/disk/resources/download?"+values.Encode(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if err := CheckAPIError(resp); err != nil {
+		return nil, err
+	}
+	defer CheckClose(resp.Body, &err)
+
+	ur, err := ParseDownloadResponse(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return ur, nil
+}
+
+// ParseDownloadResponse tries to read and parse DownloadResponse struct.
+func ParseDownloadResponse(data io.Reader) (*DownloadResponse, error) {
+	dec := json.NewDecoder(data)
+	var ur DownloadResponse
+
+	if err := dec.Decode(&ur); err == io.EOF {
+		// ok
+	} else if err != nil {
+		return nil, err
+	}
+
+	// TODO: check if there is any trash data after JSON and crash if there is.
+
+	return &ur, nil
+}

yandex/api/error.go

@@ -0,0 +1,84 @@
+package src
+
+//from yadisk
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// ErrorResponse represents erroneous API response.
+// Implements go's built in `error`.
+type ErrorResponse struct {
+	ErrorName   string `json:"error"`
+	Description string `json:"description"`
+	Message     string `json:"message"`
+
+	StatusCode int `json:""`
+}
+
+func (e *ErrorResponse) Error() string {
+	return fmt.Sprintf("[%d - %s] %s (%s)", e.StatusCode, e.ErrorName, e.Description, e.Message)
+}
+
+// ProccessErrorResponse tries to represent data passed as
+// an ErrorResponse object.
+func ProccessErrorResponse(data io.Reader) (*ErrorResponse, error) {
+	dec := json.NewDecoder(data)
+	var errorResponse ErrorResponse
+
+	if err := dec.Decode(&errorResponse); err == io.EOF {
+		// ok
+	} else if err != nil {
+		return nil, err
+	}
+
+	// TODO: check if there is any trash data after JSON and crash if there is.
+
+	return &errorResponse, nil
+}
+
+// CheckAPIError is a convenient function to turn erroneous
+// API response into go error.
+func CheckAPIError(resp *http.Response) error {
+	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
+		return nil
+	}
+
+	errorResponse, err := ProccessErrorResponse(resp.Body)
+	if err != nil {
+		return err
+	}
+	errorResponse.StatusCode = resp.StatusCode
+
+	defer CheckClose(resp.Body, &err)
+
+	return errorResponse
+}
+
+// ProccessErrorString tries to represent data passed as
+// an ErrorResponse object.
+func ProccessErrorString(data string) (*ErrorResponse, error) {
+	var errorResponse ErrorResponse
+	if err := json.Unmarshal([]byte(data), &errorResponse); err == nil {
+		// ok
+	} else if err != nil {
+		return nil, err
+	}
+
+	// TODO: check if there is any trash data after JSON and crash if there is.
+
+	return &errorResponse, nil
+}
+
+// ParseAPIError Parse json error response from API
+func (c *Client) ParseAPIError(jsonErr string) (string, error) { //ErrorName
+	errorResponse, err := ProccessErrorString(jsonErr)
+	if err != nil {
+		return err.Error(), err
+	}
+
+	return errorResponse.ErrorName, nil
+}

yandex/api/errors.go

@@ -0,0 +1,14 @@
+package src
+
+import "encoding/json"
+
+//DiskClientError struct
+type DiskClientError struct {
+	Description string `json:"Description"`
+	Code        string `json:"Error"`
+}
+
+func (e DiskClientError) Error() string {
+	b, _ := json.Marshal(e)
+	return string(b)
+}

yandex/api/files_resource_list.go

@@ -0,0 +1,8 @@
+package src
+
+// FilesResourceListResponse struct is returned by the API for requests.
+type FilesResourceListResponse struct {
+	Items  []ResourceInfoResponse `json:"items"`
+	Limit  *uint64                `json:"limit"`
+	Offset *uint64                `json:"offset"`
+}

yandex/api/flat_file_list_request.go

@@ -0,0 +1,78 @@
+package src
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+// FlatFileListRequest struct client for FlatFileList Request
+type FlatFileListRequest struct {
+	client      *Client
+	HTTPRequest *HTTPRequest
+}
+
+// FlatFileListRequestOptions struct - options for request
+type FlatFileListRequestOptions struct {
+	MediaType   []MediaType
+	Limit       *uint32
+	Offset      *uint32
+	Fields      []string
+	PreviewSize *PreviewSize
+	PreviewCrop *bool
+}
+
+// Request get request
+func (req *FlatFileListRequest) Request() *HTTPRequest {
+	return req.HTTPRequest
+}
+
+// NewFlatFileListRequest create new FlatFileList Request
+func (c *Client) NewFlatFileListRequest(options ...FlatFileListRequestOptions) *FlatFileListRequest {
+	var parameters = make(map[string]interface{})
+	if len(options) > 0 {
+		opt := options[0]
+		if opt.Limit != nil {
+			parameters["limit"] = *opt.Limit
+		}
+		if opt.Offset != nil {
+			parameters["offset"] = *opt.Offset
+		}
+		if opt.Fields != nil {
+			parameters["fields"] = strings.Join(opt.Fields, ",")
+		}
+		if opt.PreviewSize != nil {
+			parameters["preview_size"] = opt.PreviewSize.String()
+		}
+		if opt.PreviewCrop != nil {
+			parameters["preview_crop"] = *opt.PreviewCrop
+		}
+		if opt.MediaType != nil {
+			var strMediaTypes = make([]string, len(opt.MediaType))
+			for i, t := range opt.MediaType {
+				strMediaTypes[i] = t.String()
+			}
+			parameters["media_type"] = strings.Join(strMediaTypes, ",")
+		}
+	}
+	return &FlatFileListRequest{
+		client:      c,
+		HTTPRequest: createGetRequest(c, "/resources/files", parameters),
+	}
+}
+
+// Exec run FlatFileList Request
+func (req *FlatFileListRequest) Exec() (*FilesResourceListResponse, error) {
+	data, err := req.Request().run(req.client)
+	if err != nil {
+		return nil, err
+	}
+	var info FilesResourceListResponse
+	err = json.Unmarshal(data, &info)
+	if err != nil {
+		return nil, err
+	}
+	if cap(info.Items) == 0 {
+		info.Items = []ResourceInfoResponse{}
+	}
+	return &info, nil
+}

yandex/api/http_request.go

@@ -0,0 +1,28 @@
+package src
+
+// HTTPRequest struct
+type HTTPRequest struct {
+	Method     string
+	Path       string
+	Parameters map[string]interface{}
+	Headers    map[string][]string
+}
+
+func createGetRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest {
+	return createRequest(client, "GET", path, params)
+}
+
+func createPostRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest {
+	return createRequest(client, "POST", path, params)
+}
+
+func createRequest(client *Client, method string, path string, parameters map[string]interface{}) *HTTPRequest {
+	var headers = make(map[string][]string)
+	headers["Authorization"] = []string{"OAuth " + client.token}
+	return &HTTPRequest{
+		Method:     method,
+		Path:       path,
+		Parameters: parameters,
+		Headers:    headers,
+	}
+}

yandex/api/last_uploaded_resource_list.go

@@ -0,0 +1,7 @@
+package src
+
+// LastUploadedResourceListResponse struct
+type LastUploadedResourceListResponse struct {
+	Items []ResourceInfoResponse `json:"items"`
+	Limit *uint64                `json:"limit"`
+}

yandex/api/last_uploaded_resource_list_request.go

@@ -0,0 +1,74 @@
+package src
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+// LastUploadedResourceListRequest struct
+type LastUploadedResourceListRequest struct {
+	client      *Client
+	HTTPRequest *HTTPRequest
+}
+
+// LastUploadedResourceListRequestOptions struct
+type LastUploadedResourceListRequestOptions struct {
+	MediaType   []MediaType
+	Limit       *uint32
+	Fields      []string
+	PreviewSize *PreviewSize
+	PreviewCrop *bool
+}
+
+// Request return request
+func (req *LastUploadedResourceListRequest) Request() *HTTPRequest {
+	return req.HTTPRequest
+}
+
+// NewLastUploadedResourceListRequest create new LastUploadedResourceList Request
+func (c *Client) NewLastUploadedResourceListRequest(options ...LastUploadedResourceListRequestOptions) *LastUploadedResourceListRequest {
+	var parameters = make(map[string]interface{})
+	if len(options) > 0 {
+		opt := options[0]
+		if opt.Limit != nil {
+			parameters["limit"] = opt.Limit
+		}
+		if opt.Fields != nil {
+			parameters["fields"] = strings.Join(opt.Fields, ",")
+		}
+		if opt.PreviewSize != nil {
+			parameters["preview_size"] = opt.PreviewSize.String()
+		}
+		if opt.PreviewCrop != nil {
+			parameters["preview_crop"] = opt.PreviewCrop
+		}
+		if opt.MediaType != nil {
+			var strMediaTypes = make([]string, len(opt.MediaType))
+			for i, t := range opt.MediaType {
+				strMediaTypes[i] = t.String()
+			}
+			parameters["media_type"] = strings.Join(strMediaTypes, ",")
+		}
+	}
+	return &LastUploadedResourceListRequest{
+		client:      c,
+		HTTPRequest: createGetRequest(c, "/resources/last-uploaded", parameters),
+	}
+}
+
+// Exec run LastUploadedResourceList Request
+func (req *LastUploadedResourceListRequest) Exec() (*LastUploadedResourceListResponse, error) {
+	data, err := req.Request().run(req.client)
+	if err != nil {
+		return nil, err
+	}
+	var info LastUploadedResourceListResponse
+	err = json.Unmarshal(data, &info)
+	if err != nil {
+		return nil, err
+	}
+	if cap(info.Items) == 0 {
+		info.Items = []ResourceInfoResponse{}
+	}
+	return &info, nil
+}

yandex/api/media_type.go

@@ -0,0 +1,144 @@
+package src
+
+// MediaType struct - media types
+type MediaType struct {
+	mediaType string
+}
+
+// Audio - media type
+func (m *MediaType) Audio() *MediaType {
+	return &MediaType{
+		mediaType: "audio",
+	}
+}
+
+// Backup - media type
+func (m *MediaType) Backup() *MediaType {
+	return &MediaType{
+		mediaType: "backup",
+	}
+}
+
+// Book - media type
+func (m *MediaType) Book() *MediaType {
+	return &MediaType{
+		mediaType: "book",
+	}
+}
+
+// Compressed - media type
+func (m *MediaType) Compressed() *MediaType {
+	return &MediaType{
+		mediaType: "compressed",
+	}
+}
+
+// Data - media type
+func (m *MediaType) Data() *MediaType {
+	return &MediaType{
+		mediaType: "data",
+	}
+}
+
+// Development - media type
+func (m *MediaType) Development() *MediaType {
+	return &MediaType{
+		mediaType: "development",
+	}
+}
+
+// Diskimage - media type
+func (m *MediaType) Diskimage() *MediaType {
+	return &MediaType{
+		mediaType: "diskimage",
+	}
+}
+
+// Document - media type
+func (m *MediaType) Document() *MediaType {
+	return &MediaType{
+		mediaType: "document",
+	}
+}
+
+// Encoded - media type
+func (m *MediaType) Encoded() *MediaType {
+	return &MediaType{
+		mediaType: "encoded",
+	}
+}
+
+// Executable - media type
+func (m *MediaType) Executable() *MediaType {
+	return &MediaType{
+		mediaType: "executable",
+	}
+}
+
+// Flash - media type
+func (m *MediaType) Flash() *MediaType {
+	return &MediaType{
+		mediaType: "flash",
+	}
+}
+
+// Font - media type
+func (m *MediaType) Font() *MediaType {
+	return &MediaType{
+		mediaType: "font",
+	}
+}
+
+// Image - media type
+func (m *MediaType) Image() *MediaType {
+	return &MediaType{
+		mediaType: "image",
+	}
+}
+
+// Settings - media type
+func (m *MediaType) Settings() *MediaType {
+	return &MediaType{
+		mediaType: "settings",
+	}
+}
+
+// Spreadsheet - media type
+func (m *MediaType) Spreadsheet() *MediaType {
+	return &MediaType{
+		mediaType: "spreadsheet",
+	}
+}
+
+// Text - media type
+func (m *MediaType) Text() *MediaType {
+	return &MediaType{
+		mediaType: "text",
+	}
+}
+
+// Unknown - media type
+func (m *MediaType) Unknown() *MediaType {
+	return &MediaType{
+		mediaType: "unknown",
+	}
+}
+
+// Video - media type
+func (m *MediaType) Video() *MediaType {
+	return &MediaType{
+		mediaType: "video",
+	}
+}
+
+// Web - media type
+func (m *MediaType) Web() *MediaType {
+	return &MediaType{
+		mediaType: "web",
+	}
+}
+
+// String - media type
+func (m *MediaType) String() string {
+	return m.mediaType
+}

yandex/api/mkdir.go

@@ -0,0 +1,21 @@
+package src
+
+import (
+	"net/url"
+)
+
+// Mkdir will make specified folder on Yandex Disk
+func (c *Client) Mkdir(remotePath string) (int, string, error) {
+
+	values := url.Values{}
+	values.Add("path", remotePath) // only one current folder will be created. Not all the folders in the path.
+	urlPath := "/v1/disk/resources?" + values.Encode()
+	fullURL := RootAddr
+	if urlPath[:1] != "/" {
+		fullURL += "/" + urlPath
+	} else {
+		fullURL += urlPath
+	}
+
+	return c.PerformMkdir(fullURL)
+}

yandex/api/performdelete.go

@@ -0,0 +1,34 @@
+package src
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// PerformDelete does the actual delete via DELETE request.
+func (c *Client) PerformDelete(url string) error {
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil {
+		return err
+	}
+
+	//set access token and headers
+	c.setRequestScope(req)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	//204 - resource deleted.
+	//202 - folder not empty, content will be deleted soon (async delete).
+	if resp.StatusCode != 204 && resp.StatusCode != 202 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("delete error [%d]: %s", resp.StatusCode, string(body[:]))
+	}
+	return nil
+}

yandex/api/performdownload.go

@@ -0,0 +1,33 @@
+package src
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+// PerformDownload does the actual download via unscoped PUT request.
+func (c *Client) PerformDownload(url string) (io.ReadCloser, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	//c.setRequestScope(req)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer CheckClose(resp.Body, &err)
+		return nil, fmt.Errorf("download error [%d]: %s", resp.StatusCode, string(body[:]))
+	}
+	return resp.Body, err
+}

yandex/api/performmkdir.go

@@ -0,0 +1,33 @@
+package src
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// PerformMkdir does the actual mkdir via PUT request.
+func (c *Client) PerformMkdir(url string) (int, string, error) {
+	req, err := http.NewRequest("PUT", url, nil)
+	if err != nil {
+		return 0, "", err
+	}
+
+	//set access token and headers
+	c.setRequestScope(req)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, "", err
+	}
+
+	if resp.StatusCode != 201 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return 0, "", err
+		}
+		//third parameter is the json error response body
+		return resp.StatusCode, string(body[:]), fmt.Errorf("Create Folder error [%d]: %s", resp.StatusCode, string(body[:]))
+	}
+	return resp.StatusCode, "", nil
+}

yandex/api/performupload.go

@@ -0,0 +1,36 @@
+package src
+
+//from yadisk
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+// PerformUpload does the actual upload via unscoped PUT request.
+func (c *Client) PerformUpload(url string, data io.Reader) error {
+	req, err := http.NewRequest("PUT", url, data)
+	if err != nil {
+		return err
+	}
+
+	//c.setRequestScope(req)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer CheckClose(resp.Body, &err)
+
+	if resp.StatusCode != 201 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		return fmt.Errorf("upload error [%d]: %s", resp.StatusCode, string(body[:]))
+	}
+	return nil
+}

yandex/api/preview_size.go

@@ -0,0 +1,75 @@
+package src
+
+import "fmt"
+
+// PreviewSize struct
+type PreviewSize struct {
+	size string
+}
+
+// PredefinedSizeS - set preview size
+func (s *PreviewSize) PredefinedSizeS() *PreviewSize {
+	return &PreviewSize{
+		size: "S",
+	}
+}
+
+// PredefinedSizeM - set preview size
+func (s *PreviewSize) PredefinedSizeM() *PreviewSize {
+	return &PreviewSize{
+		size: "M",
+	}
+}
+
+// PredefinedSizeL - set preview size
+func (s *PreviewSize) PredefinedSizeL() *PreviewSize {
+	return &PreviewSize{
+		size: "L",
+	}
+}
+
+// PredefinedSizeXL - set preview size
+func (s *PreviewSize) PredefinedSizeXL() *PreviewSize {
+	return &PreviewSize{
+		size: "XL",
+	}
+}
+
+// PredefinedSizeXXL - set preview size
+func (s *PreviewSize) PredefinedSizeXXL() *PreviewSize {
+	return &PreviewSize{
+		size: "XXL",
+	}
+}
+
+// PredefinedSizeXXXL - set preview size
+func (s *PreviewSize) PredefinedSizeXXXL() *PreviewSize {
+	return &PreviewSize{
+		size: "XXXL",
+	}
+}
+
+// ExactWidth - set preview size
+func (s *PreviewSize) ExactWidth(width uint32) *PreviewSize {
+	return &PreviewSize{
+		size: fmt.Sprintf("%dx", width),
+	}
+}
+
+// ExactHeight - set preview size
+func (s *PreviewSize) ExactHeight(height uint32) *PreviewSize {
+	return &PreviewSize{
+		size: fmt.Sprintf("x%d", height),
+	}
+}
+
+// ExactSize - set preview size
+func (s *PreviewSize) ExactSize(width uint32, height uint32) *PreviewSize {
+	return &PreviewSize{
+		size: fmt.Sprintf("%dx%d", width, height),
+	}
+}
+
+func (s *PreviewSize) String() string {
+	return s.size
+}

yandex/api/resource.go

@@ -0,0 +1,19 @@
+package src
+
+//ResourceInfoResponse struct is returned by the API for metedata requests.
+type ResourceInfoResponse struct {
+	PublicKey        string                 `json:"public_key"`
+	Name             string                 `json:"name"`
+	Created          string                 `json:"created"`
+	CustomProperties map[string]interface{} `json:"custom_properties"`
+	Preview          string                 `json:"preview"`
+	PublicURL        string                 `json:"public_url"`
+	OriginPath       string                 `json:"origin_path"`
+	Modified         string                 `json:"modified"`
+	Path             string                 `json:"path"`
+	Md5              string                 `json:"md5"`
+	ResourceType     string                 `json:"type"`
+	MimeType         string                 `json:"mime_type"`
+	Size             uint64                 `json:"size"`
+	Embedded         *ResourceListResponse  `json:"_embedded"`
+}

yandex/api/resource_info_request.go

@@ -0,0 +1,45 @@
+package src
+
+import "encoding/json"
+
+// ResourceInfoRequest struct
+type ResourceInfoRequest struct {
+	client      *Client
+	HTTPRequest *HTTPRequest
+}
+
+// Request of ResourceInfoRequest
+func (req *ResourceInfoRequest) Request() *HTTPRequest {
+	return req.HTTPRequest
+}
+
+// NewResourceInfoRequest create new ResourceInfo Request
+func (c *Client) NewResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *ResourceInfoRequest {
+	return &ResourceInfoRequest{
+		client:      c,
+		HTTPRequest: createResourceInfoRequest(c, "/resources", path, options...),
+	}
+}
+
+// Exec run ResourceInfo Request
+func (req *ResourceInfoRequest) Exec() (*ResourceInfoResponse, error) {
+	data, err := req.Request().run(req.client)
+	if err != nil {
+		return nil, err
+	}
+
+	var info ResourceInfoResponse
+	err = json.Unmarshal(data, &info)
+	if err != nil {
+		return nil, err
+	}
+	if info.CustomProperties == nil {
+		info.CustomProperties = make(map[string]interface{})
+	}
+	if info.Embedded != nil {
+		if cap(info.Embedded.Items) == 0 {
+			info.Embedded.Items = []ResourceInfoResponse{}
+		}
+	}
+	return &info, nil
+}

yandex/api/resource_info_request_helpers.go

@@ -0,0 +1,33 @@
+package src
+
+import "strings"
+
+func createResourceInfoRequest(c *Client,
+	apiPath string,
+	path string,
+	options ...ResourceInfoRequestOptions) *HTTPRequest {
+	var parameters = make(map[string]interface{})
+	parameters["path"] = path
+	if len(options) > 0 {
+		opt := options[0]
+		if opt.SortMode != nil {
+			parameters["sort"] = opt.SortMode.String()
+		}
+		if opt.Limit != nil {
+			parameters["limit"] = opt.Limit
+		}
+		if opt.Offset != nil {
+			parameters["offset"] = opt.Offset
+		}
+		if opt.Fields != nil {
+			parameters["fields"] = strings.Join(opt.Fields, ",")
+		}
+		if opt.PreviewSize != nil {
+			parameters["preview_size"] = opt.PreviewSize.String()
+		}
+		if opt.PreviewCrop != nil {
+			parameters["preview_crop"] = opt.PreviewCrop
+		}
+	}
+	return createGetRequest(c, apiPath, parameters)
+}

yandex/api/resource_info_request_options.go

@@ -0,0 +1,11 @@
+package src
+
+// ResourceInfoRequestOptions struct
+type ResourceInfoRequestOptions struct {
+	SortMode    *SortMode
+	Limit       *uint32
+	Offset      *uint32
+	Fields      []string
+	PreviewSize *PreviewSize
+	PreviewCrop *bool
+}

yandex/api/resource_list.go

@@ -0,0 +1,12 @@
+package src
+
+// ResourceListResponse struct
+type ResourceListResponse struct {
+	Sort      *SortMode              `json:"sort"`
+	PublicKey string                 `json:"public_key"`
+	Items     []ResourceInfoResponse `json:"items"`
+	Path      string                 `json:"path"`
+	Limit     *uint64                `json:"limit"`
+	Offset    *uint64                `json:"offset"`
+	Total     *uint64                `json:"total"`
+}

yandex/api/sort_mode.go

@@ -0,0 +1,79 @@
+package src
+
+import "strings"
+
+// SortMode struct - sort mode
+type SortMode struct {
+	mode string
+}
+
+// Default - sort mode
+func (m *SortMode) Default() *SortMode {
+	return &SortMode{
+		mode: "",
+	}
+}
+
+// ByName - sort mode
+func (m *SortMode) ByName() *SortMode {
+	return &SortMode{
+		mode: "name",
+	}
+}
+
+// ByPath - sort mode
+func (m *SortMode) ByPath() *SortMode {
+	return &SortMode{
+		mode: "path",
+	}
+}
+
+// ByCreated - sort mode
+func (m *SortMode) ByCreated() *SortMode {
+	return &SortMode{
+		mode: "created",
+	}
+}
+
+// ByModified - sort mode
+func (m *SortMode) ByModified() *SortMode {
+	return &SortMode{
+		mode: "modified",
+	}
+}
+
+// BySize - sort mode
+func (m *SortMode) BySize() *SortMode {
+	return &SortMode{
+		mode: "size",
+	}
+}
+
+// Reverse - sort mode
+func (m *SortMode) Reverse() *SortMode {
+	if strings.HasPrefix(m.mode, "-") {
+		return &SortMode{
+			mode: m.mode[1:],
+		}
+	}
+	return &SortMode{
+		mode: "-" + m.mode,
+	}
+}
+
+func (m *SortMode) String() string {
+	return m.mode
+}
+
+// UnmarshalJSON sort mode
+func (m *SortMode) UnmarshalJSON(value []byte) error {
+	if value == nil || len(value) == 0 {
+		m.mode = ""
+		return nil
+	}
+	m.mode = string(value)
+	if strings.HasPrefix(m.mode, "\"") && strings.HasSuffix(m.mode, "\"") {
+		m.mode = m.mode[1 : len(m.mode)-1]
+	}
+	return nil
+}

yandex/api/trash_resource_info_request.go

@@ -0,0 +1,45 @@
+package src
+
+import "encoding/json"
+
+// TrashResourceInfoRequest struct
+type TrashResourceInfoRequest struct {
+	client      *Client
+	HTTPRequest *HTTPRequest
+}
+
+// Request of TrashResourceInfoRequest struct
+func (req *TrashResourceInfoRequest) Request() *HTTPRequest {
+	return req.HTTPRequest
+}
+
+// NewTrashResourceInfoRequest create new TrashResourceInfo Request
+func (c *Client) NewTrashResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *TrashResourceInfoRequest {
+	return &TrashResourceInfoRequest{
+		client:      c,
+		HTTPRequest: createResourceInfoRequest(c, "/trash/resources", path, options...),
+	}
+}
+
+// Exec run TrashResourceInfo Request
+func (req *TrashResourceInfoRequest) Exec() (*ResourceInfoResponse, error) {
+	data, err := req.Request().run(req.client)
+	if err != nil {
+		return nil, err
+	}
+
+	var info ResourceInfoResponse
+	err = json.Unmarshal(data, &info)
+	if err != nil {
+		return nil, err
+	}
+	if info.CustomProperties == nil {
+		info.CustomProperties = make(map[string]interface{})
+	}
+	if info.Embedded != nil {
+		if cap(info.Embedded.Items) == 0 {
+			info.Embedded.Items = []ResourceInfoResponse{}
+		}
+	}
+	return &info, nil
+}

yandex/api/upload.go

@@ -0,0 +1,75 @@
+package src
+
+//from yadisk
+
+import (
+	"encoding/json"
+	"io"
+	"net/url"
+	"strconv"
+)
+
+// UploadResponse struct is returned by the API for upload request.
+type UploadResponse struct {
+	HRef      string `json:"href"`
+	Method    string `json:"method"`
+	Templated bool   `json:"templated"`
+}
+
+// Upload will put specified data to Yandex.Disk.
+func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool) error {
+	ur, err := c.UploadRequest(remotePath, overwrite)
+	if err != nil {
+		return err
+	}
+
+	if err := c.PerformUpload(ur.HRef, data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UploadRequest will make an upload request and return a URL to upload data to.
+func (c *Client) UploadRequest(remotePath string, overwrite bool) (*UploadResponse, error) {
+	values := url.Values{}
+	values.Add("path", remotePath)
+	values.Add("overwrite", strconv.FormatBool(overwrite))
+
+	req, err := c.scopedRequest("GET", "/v1/disk/resources/upload?"+values.Encode(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if err := CheckAPIError(resp); err != nil {
+		return nil, err
+	}
+	defer CheckClose(resp.Body, &err)
+
+	ur, err := ParseUploadResponse(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return ur, nil
+}
+
+// ParseUploadResponse tries to read and parse UploadResponse struct.
+func ParseUploadResponse(data io.Reader) (*UploadResponse, error) {
+	dec := json.NewDecoder(data)
+	var ur UploadResponse
+
+	if err := dec.Decode(&ur); err == io.EOF {
+		// ok
+	} else if err != nil {
+		return nil, err
+	}
+
+	// TODO: check if there is any trash data after JSON and crash if there is.
+
+	return &ur, nil
+}

yandex/yandex.go

@@ -0,0 +1,541 @@
+// Package yandex provides an interface to the Yandex Disk storage.
+//
+// dibu28 <dibu28@gmail.com> github.com/dibu28
+package yandex
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/oauthutil"
+	yandex "github.com/ncw/rclone/yandex/api"
+	"golang.org/x/oauth2"
+)
+
+//oAuth
+const (
+	rcloneClientID     = "ac39b43b9eba4cae8ffb788c06d816a8"
+	rcloneClientSecret = "k8jKzZnMmM+Wx5jAksPAwYKPgImOiN+FhNKD09KBg9A="
+)
+
+// Globals
+var (
+	// Description of how to auth for this app
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
+			TokenURL: "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: fs.Reveal(rcloneClientSecret),
+		RedirectURL:  oauthutil.RedirectURL,
+	}
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.Info{
+		Name:  "yandex",
+		NewFs: NewFs,
+		Config: func(name string) {
+			err := oauthutil.Config(name, oauthConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
+		Options: []fs.Option{{
+			Name: oauthutil.ConfigClientID,
+			Help: "Yandex Client Id - leave blank normally.",
+		}, {
+			Name: oauthutil.ConfigClientSecret,
+			Help: "Yandex Client Secret - leave blank normally.",
+		}},
+	})
+}
+
+// Fs represents a remote yandex
+type Fs struct {
+	name       string
+	yd         *yandex.Client // client for rest api
+	root       string         //root path
+	diskRoot   string         //root path with "disk:/" container name
+	mkdircache map[string]int
+}
+
+// Object describes a swift object
+type Object struct {
+	fs      *Fs       // what this object is part of
+	remote  string    // The remote path
+	md5sum  string    // The MD5Sum of the object
+	bytes   uint64    // Bytes in the object
+	modTime time.Time // Modified time of the object
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Yandex %s", f.root)
+}
+
+// read access token from ConfigFile string
+func getAccessToken(name string) (*oauth2.Token, error) {
+	// Read the token from the config file
+	tokenConfig := fs.ConfigFile.MustValue(name, "token")
+	//Get access token from config string
+	decoder := json.NewDecoder(strings.NewReader(tokenConfig))
+	var result *oauth2.Token
+	err := decoder.Decode(&result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(name, root string) (fs.Fs, error) {
+	//read access token from config
+	token, err := getAccessToken(name)
+	if err != nil {
+		return nil, err
+	}
+
+	//create new client
+	yandexDisk := yandex.NewClient(token.AccessToken)
+
+	f := &Fs{
+		yd: yandexDisk,
+	}
+
+	f.setRoot(root)
+
+	//limited fs
+	// Check to see if the object exists and is a file
+	//request object meta info
+	var opt2 yandex.ResourceInfoRequestOptions
+	if ResourceInfoResponse, err := yandexDisk.NewResourceInfoRequest(root, opt2).Exec(); err != nil {
+		//return err
+	} else {
+		if ResourceInfoResponse.ResourceType == "file" {
+			//limited fs
+			remote := path.Base(root)
+			f.setRoot(path.Dir(root))
+
+			obj := f.newFsObjectWithInfo(remote, ResourceInfoResponse)
+			// return a Fs Limited to this object
+			return fs.NewLimited(f, obj), nil
+		}
+	}
+
+	return f, nil
+}
+
+// Sets root in f
+func (f *Fs) setRoot(root string) {
+	//Set root path
+	f.root = strings.Trim(root, "/")
+	//Set disk root path.
+	//Adding "disk:" to root path as all paths on disk start with it
+	var diskRoot = ""
+	if f.root == "" {
+		diskRoot = "disk:/"
+	} else {
+		diskRoot = "disk:/" + f.root + "/"
+	}
+	f.diskRoot = diskRoot
+}
+
+// list the objects into the function supplied
+//
+// If directories is set it only sends directories
+func (f *Fs) list(directories bool, fn func(string, yandex.ResourceInfoResponse)) {
+	//request files list. list is divided into pages. We send request for each page
+	//items per page is limited by limit
+	//TODO may be add config parameter for the items per page limit
+	var limit uint32 = 1000 // max number of object per request
+	var itemsCount uint32   //number of items per page in response
+	var offset uint32       //for the next page of request
+	// yandex disk api request options
+	var opt yandex.FlatFileListRequestOptions
+	opt.Limit = &limit
+	opt.Offset = &offset
+	//query each page of list until itemCount is less then limit
+	for {
+		//send request
+		info, err := f.yd.NewFlatFileListRequest(opt).Exec()
+		if err != nil {
+			fs.Stats.Error()
+			fs.ErrorLog(f, "Couldn't list: %s", err)
+			return
+		}
+		itemsCount = uint32(len(info.Items))
+
+		//list files
+		for _, item := range info.Items {
+			// filter file list and get only files we need
+			if strings.HasPrefix(item.Path, f.diskRoot) {
+				//trim root folder from filename
+				var name = strings.TrimPrefix(item.Path, f.diskRoot)
+				fn(name, item)
+			}
+		}
+
+		//offset for the next page of items
+		offset += itemsCount
+		//check if we reached end of list
+		if itemsCount < limit {
+			break
+		}
+	}
+}
+
+// List walks the path returning a channel of FsObjects
+func (f *Fs) List() fs.ObjectsChan {
+	out := make(fs.ObjectsChan, fs.Config.Checkers)
+	// List the objects
+	go func() {
+		defer close(out)
+		f.list(false, func(remote string, object yandex.ResourceInfoResponse) {
+			if fs := f.newFsObjectWithInfo(remote, &object); fs != nil {
+				out <- fs
+			}
+		})
+	}()
+	return out
+}
+
+// NewFsObject returns an Object from a path
+//
+// May return nil if an error occurred
+func (f *Fs) NewFsObject(remote string) fs.Object {
+	return f.newFsObjectWithInfo(remote, nil)
+}
+
+// Return an FsObject from a path
+//
+// May return nil if an error occurred
+func (f *Fs) newFsObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) fs.Object {
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	if info != nil {
+		o.setMetaData(info)
+	} else {
+		err := o.readMetaData()
+		if err != nil {
+			fs.ErrorLog(f, "Couldn't get object '%s' metadata: %s", o.remotePath(), err)
+			return nil
+		}
+	}
+	return o
+}
+
+// setMetaData sets the fs data from a storage.Object
+func (o *Object) setMetaData(info *yandex.ResourceInfoResponse) {
+	o.bytes = info.Size
+	o.md5sum = info.Md5
+
+	if info.CustomProperties["rclone_modified"] == nil {
+		//read modTime from Modified property of object
+		t, err := time.Parse(time.RFC3339Nano, info.Modified)
+		if err != nil {
+			return
+		}
+		o.modTime = t
+	} else {
+		// interface{} to string type assertion
+		if modtimestr, ok := info.CustomProperties["rclone_modified"].(string); ok {
+			//read modTime from rclone_modified custom_property of object
+			t, err := time.Parse(time.RFC3339Nano, modtimestr)
+			if err != nil {
+				return
+			}
+			o.modTime = t
+		} else {
+			return //if it is not a string
+		}
+	}
+}
+
+// readMetaData gets the info if it hasn't already been fetched
+func (o *Object) readMetaData() (err error) {
+	//request meta info
+	var opt2 yandex.ResourceInfoRequestOptions
+	ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec()
+	if err != nil {
+		return err
+	}
+	o.setMetaData(ResourceInfoResponse)
+	return nil
+}
+
+// ListDir walks the path returning a channel of FsObjects
+func (f *Fs) ListDir() fs.DirChan {
+	out := make(fs.DirChan, fs.Config.Checkers)
+	go func() {
+		defer close(out)
+
+		//request object meta info
+		var opt yandex.ResourceInfoRequestOptions
+		ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(f.diskRoot, opt).Exec()
+		if err != nil {
+			return
+		}
+		if ResourceInfoResponse.ResourceType == "dir" {
+			//list all subdirs
+			for _, element := range ResourceInfoResponse.Embedded.Items {
+				if element.ResourceType == "dir" {
+					t, err := time.Parse(time.RFC3339Nano, element.Modified)
+					if err != nil {
+						return
+					}
+					out <- &fs.Dir{
+						Name:  element.Name,
+						When:  t,
+						Bytes: int64(element.Size),
+						Count: -1,
+					}
+				}
+			}
+		}
+
+	}()
+	return out
+}
+
+// Put the object
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
+	o := &Object{
+		fs:      f,
+		remote:  remote,
+		bytes:   uint64(size),
+		modTime: modTime,
+	}
+	//TODO maybe read metadata after upload to check if file uploaded successfully
+	return o, o.Update(in, modTime, size)
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *Fs) Mkdir() error {
+	return mkDirFullPath(f.yd, f.diskRoot)
+}
+
+// Rmdir deletes the container
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir() error {
+	return f.purgeCheck(true)
+}
+
+// purgeCheck remotes the root directory, if check is set then it
+// refuses to do so if it has anything in
+func (f *Fs) purgeCheck(check bool) error {
+	if check {
+		//to comply with rclone logic we check if the directory is empty before delete.
+		//send request to get list of objects in this directory.
+		var opt yandex.ResourceInfoRequestOptions
+		ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(f.diskRoot, opt).Exec()
+		if err != nil {
+			return fmt.Errorf("Rmdir failed: %s", err)
+		}
+		if len(ResourceInfoResponse.Embedded.Items) != 0 {
+			return fmt.Errorf("Rmdir failed: Directory not empty")
+		}
+	}
+	//delete directory
+	return f.yd.Delete(f.diskRoot, true)
+}
+
+// Precision return the precision of this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge() error {
+	return f.purgeCheck(false)
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Fs {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Md5sum returns the Md5sum of an object returning a lowercase hex string
+func (o *Object) Md5sum() (string, error) {
+	return o.md5sum, nil
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	var size = int64(o.bytes) //need to cast from uint64 in yandex disk to int64 in rclone. can cause overflow
+	return size
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime() time.Time {
+	err := o.readMetaData()
+	if err != nil {
+		fs.Log(o, "Failed to read metadata: %s", err)
+		return time.Now()
+	}
+	return o.modTime
+}
+
+// Open an object for read
+func (o *Object) Open() (in io.ReadCloser, err error) {
+	return o.fs.yd.Download(o.remotePath())
+}
+
+// Remove an object
+func (o *Object) Remove() error {
+	return o.fs.yd.Delete(o.remotePath(), true)
+}
+
+// SetModTime sets the modification time of the local fs object
+//
+// Commits the datastore
+func (o *Object) SetModTime(modTime time.Time) {
+	remote := o.remotePath()
+	//set custom_property 'rclone_modified' of object to modTime
+	err := o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Storable returns whether this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Returns the remote path for the object
+func (o *Object) remotePath() string {
+	return o.fs.diskRoot + o.remote
+}
+
+// Update the already existing object
+//
+// Copy the reader into the object updating modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
+	remote := o.remotePath()
+	//create full path to file before upload.
+	err1 := mkDirFullPath(o.fs.yd, remote)
+	if err1 != nil {
+		return err1
+	}
+	//upload file
+	overwrite := true //overwrite existing file
+	err := o.fs.yd.Upload(in, remote, overwrite)
+	if err == nil {
+		//if file uploaded sucessfuly then return metadata
+		o.bytes = uint64(size)
+		o.modTime = modTime
+		o.md5sum = "" // according to unit tests after put the md5 is empty.
+		//and set modTime of uploaded file
+		o.SetModTime(modTime)
+	}
+	return err
+}
+
+// utility funcs-------------------------------------------------------------------
+
+// mkDirExecute execute mkdir
+func mkDirExecute(client *yandex.Client, path string) (int, string, error) {
+	statusCode, jsonErrorString, err := client.Mkdir(path)
+	if statusCode == 409 { // dir already exist
+		return statusCode, jsonErrorString, err
+	}
+	if statusCode == 201 { // dir was created
+		return statusCode, jsonErrorString, err
+	}
+	if err != nil {
+		// error creating directory
+		log.Fatalf("Failed to create folder: %v", err)
+		return statusCode, jsonErrorString, err
+	}
+	return 0, "", nil
+}
+
+//mkDirFullPath Creates Each Directory in the path if needed. Send request once for every directory in the path.
+func mkDirFullPath(client *yandex.Client, path string) error {
+	//trim filename from path
+	dirString := strings.TrimSuffix(path, filepath.Base(path))
+	//trim "disk:/" from path
+	dirString = strings.TrimPrefix(dirString, "disk:/")
+
+	//1 Try to create directory first
+	if _, jsonErrorString, err := mkDirExecute(client, dirString); err != nil {
+		er2, _ := client.ParseAPIError(jsonErrorString)
+		if er2 != "DiskPathPointsToExistentDirectoryError" {
+			//2 if it fails then create all directories in the path from root.
+			dirs := strings.Split(dirString, "/") //path separator /
+			var mkdirpath = "/"                   //path separator /
+			for _, element := range dirs {
+				if element != "" {
+					mkdirpath += element + "/" //path separator /
+					_, _, err2 := mkDirExecute(client, mkdirpath)
+					if err2 != nil {
+						//we continue even if some directories exist.
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs     = (*Fs)(nil)
+	_ fs.Purger = (*Fs)(nil)
+	//_ fs.Copier = (*Fs)(nil)
+	_ fs.Object = (*Object)(nil)
+)

yandex/yandex_test.go

@@ -0,0 +1,56 @@
+// Test Yandex filesystem interface
+//
+// Automatically generated - DO NOT EDIT
+// Regenerate with: make gen_tests
+package yandex_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/ncw/rclone/yandex"
+)
+
+func init() {
+	fstests.NilObject = fs.Object((*yandex.Object)(nil))
+	fstests.RemoteName = "TestYandex:"
+}
+
+// Generic tests for the Fs
+func TestInit(t *testing.T)                  { fstests.TestInit(t) }
+func TestFsString(t *testing.T)              { fstests.TestFsString(t) }
+func TestFsRmdirEmpty(t *testing.T)          { fstests.TestFsRmdirEmpty(t) }
+func TestFsRmdirNotFound(t *testing.T)       { fstests.TestFsRmdirNotFound(t) }
+func TestFsMkdir(t *testing.T)               { fstests.TestFsMkdir(t) }
+func TestFsListEmpty(t *testing.T)           { fstests.TestFsListEmpty(t) }
+func TestFsListDirEmpty(t *testing.T)        { fstests.TestFsListDirEmpty(t) }
+func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
+func TestFsPutFile1(t *testing.T)            { fstests.TestFsPutFile1(t) }
+func TestFsPutFile2(t *testing.T)            { fstests.TestFsPutFile2(t) }
+func TestFsListDirFile2(t *testing.T)        { fstests.TestFsListDirFile2(t) }
+func TestFsListDirRoot(t *testing.T)         { fstests.TestFsListDirRoot(t) }
+func TestFsListRoot(t *testing.T)            { fstests.TestFsListRoot(t) }
+func TestFsListFile1(t *testing.T)           { fstests.TestFsListFile1(t) }
+func TestFsNewFsObject(t *testing.T)         { fstests.TestFsNewFsObject(t) }
+func TestFsListFile1and2(t *testing.T)       { fstests.TestFsListFile1and2(t) }
+func TestFsCopy(t *testing.T)                { fstests.TestFsCopy(t) }
+func TestFsMove(t *testing.T)                { fstests.TestFsMove(t) }
+func TestFsDirMove(t *testing.T)             { fstests.TestFsDirMove(t) }
+func TestFsRmdirFull(t *testing.T)           { fstests.TestFsRmdirFull(t) }
+func TestFsPrecision(t *testing.T)           { fstests.TestFsPrecision(t) }
+func TestObjectString(t *testing.T)          { fstests.TestObjectString(t) }
+func TestObjectFs(t *testing.T)              { fstests.TestObjectFs(t) }
+func TestObjectRemote(t *testing.T)          { fstests.TestObjectRemote(t) }
+func TestObjectMd5sum(t *testing.T)          { fstests.TestObjectMd5sum(t) }
+func TestObjectModTime(t *testing.T)         { fstests.TestObjectModTime(t) }
+func TestObjectSetModTime(t *testing.T)      { fstests.TestObjectSetModTime(t) }
+func TestObjectSize(t *testing.T)            { fstests.TestObjectSize(t) }
+func TestObjectOpen(t *testing.T)            { fstests.TestObjectOpen(t) }
+func TestObjectUpdate(t *testing.T)          { fstests.TestObjectUpdate(t) }
+func TestObjectStorable(t *testing.T)        { fstests.TestObjectStorable(t) }
+func TestLimitedFs(t *testing.T)             { fstests.TestLimitedFs(t) }
+func TestLimitedFsNotFound(t *testing.T)     { fstests.TestLimitedFsNotFound(t) }
+func TestObjectRemove(t *testing.T)          { fstests.TestObjectRemove(t) }
+func TestObjectPurge(t *testing.T)           { fstests.TestObjectPurge(t) }
+func TestFinalise(t *testing.T)              { fstests.TestFinalise(t) }