rclone/cmd/serve/restic/restic.go

// Package restic serves a remote suitable for use with restic
package restic

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/cmd/serve/httplib"
	"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/accounting"
	"github.com/ncw/rclone/fs/fserrors"
	"github.com/ncw/rclone/fs/object"
	"github.com/ncw/rclone/fs/operations"
	"github.com/ncw/rclone/fs/walk"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/http2"
)

var (
	stdio      bool
	appendOnly bool
)

func init() {
	httpflags.AddFlags(Command.Flags())
	Command.Flags().BoolVar(&stdio, "stdio", false, "run an HTTP2 server on stdin/stdout")
	Command.Flags().BoolVar(&appendOnly, "append-only", false, "disallow deletion of repository data")
}

// Command definition for cobra
var Command = &cobra.Command{
	Use:   "restic remote:path",
	Short: `Serve the remote for restic's REST API.`,
	Long: `rclone serve restic implements restic's REST backend API
over HTTP.  This allows restic to use rclone as a data storage
mechanism for cloud providers that restic does not support directly.

[Restic](https://restic.net/) is a command line program for doing
backups.

The server will log errors.  Use -v to see access logs.

--bwlimit will be respected for file transfers.  Use --stats to
control the stats printing.

### Setting up rclone for use by restic ###

First [set up a remote for your chosen cloud provider](/docs/#configure).

Once you have set up the remote, check it is working with, for example
"rclone lsd remote:".  You may have called the remote something other
than "remote:" - just substitute whatever you called it in the
following instructions.

Now start the rclone restic server

    rclone serve restic -v remote:backup

Where you can replace "backup" in the above by whatever path in the
remote you wish to use.

By default this will serve on "localhost:8080" you can change this
with use of the "--addr" flag.

You might wish to start this server on boot.

### Setting up restic to use rclone ###

Now you can [follow the restic
instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server)
on setting up restic.

Note that you will need restic 0.8.2 or later to interoperate with
rclone.

For the example above you will want to use "http://localhost:8080/" as
the URL for the REST server.

For example:

    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/
    $ export RESTIC_PASSWORD=yourpassword
    $ restic init
    created restic backend 8b1a4b56ae at rest:http://localhost:8080/
    
    Please note that knowledge of your password is required to access
    the repository. Losing your password means that your data is
    irrecoverably lost.
    $ restic backup /path/to/files/to/backup
    scan [/path/to/files/to/backup]
    scanned 189 directories, 312 files in 0:00
    [0:00] 100.00%  38.128 MiB / 38.128 MiB  501 / 501 items  0 errors  ETA 0:00 
    duration: 0:00
    snapshot 45c8fdd8 saved

#### Multiple repositories ####

Note that you can use the endpoint to host multiple repositories.  Do
this by adding a directory name or path after the URL.  Note that
these **must** end with /.  Eg

    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/
    # backup user1 stuff
    $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/
    # backup user2 stuff

` + httplib.Help,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(1, 1, command, args)
		f := cmd.NewFsSrc(args)
		cmd.Run(false, true, command, func() error {
			s := newServer(f, &httpflags.Opt)
			if stdio {
				if terminal.IsTerminal(int(os.Stdout.Fd())) {
					return errors.New("Refusing to run HTTP2 server directly on a terminal, please let restic start rclone")
				}

				conn := &StdioConn{
					stdin:  os.Stdin,
					stdout: os.Stdout,
				}

				httpSrv := &http2.Server{}
				opts := &http2.ServeConnOpts{
					Handler: http.HandlerFunc(s.handler),
				}
				httpSrv.ServeConn(conn, opts)
				return nil
			}

			s.serve()
			return nil
		})
	},
}

const (
	resticAPIV2 = "application/vnd.x.restic.rest.v2"
)

// server contains everything to run the server
type server struct {
	f   fs.Fs
	srv *httplib.Server
}

func newServer(f fs.Fs, opt *httplib.Options) *server {
	mux := http.NewServeMux()
	s := &server{
		f:   f,
		srv: httplib.NewServer(mux, opt),
	}
	mux.HandleFunc("/", s.handler)
	return s
}

// serve runs the http server - doesn't return
func (s *server) serve() {
	err := s.srv.Serve()
	if err != nil {
		fs.Errorf(s.f, "Opening listener: %v", err)
	}
	fs.Logf(s.f, "Serving restic REST API on %s", s.srv.URL())
	s.srv.Wait()
}

var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$")

// Makes a remote from a URL path.  This implements the backend layout
// required by restic.
func makeRemote(path string) string {
	path = strings.Trim(path, "/")
	parts := matchData.FindStringSubmatch(path)
	// if no data directory, layout is flat
	if parts == nil {
		return path
	}
	// otherwise map
	// data/2159dd48 to
	// data/21/2159dd48
	fileName := parts[1]
	prefix := path[:len(path)-len(fileName)]
	return prefix + fileName[:2] + "/" + fileName
}

// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Accept-Ranges", "bytes")
	w.Header().Set("Server", "rclone/"+fs.Version)

	path := r.URL.Path
	remote := makeRemote(path)
	fs.Debugf(s.f, "%s %s", r.Method, path)

	// Dispatch on path then method
	if strings.HasSuffix(path, "/") {
		switch r.Method {
		case "GET":
			s.listObjects(w, r, remote)
		case "POST":
			s.createRepo(w, r, remote)
		default:
			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		}
	} else {
		switch r.Method {
		case "GET":
			s.getObject(w, r, remote)
		case "HEAD":
			s.headObject(w, r, remote)
		case "POST":
			s.postObject(w, r, remote)
		case "DELETE":
			s.deleteObject(w, r, remote)
		default:
			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		}
	}
}

// head request the remote
func (s *server) headObject(w http.ResponseWriter, r *http.Request, remote string) {
	o, err := s.f.NewObject(remote)
	if err != nil {
		fs.Debugf(remote, "Head request error: %v", err)
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	// Set content length since we know how long the object is
	w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
}

// get the remote
func (s *server) getObject(w http.ResponseWriter, r *http.Request, remote string) {
	o, err := s.f.NewObject(remote)
	if err != nil {
		fs.Debugf(remote, "Get request error: %v", err)
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	// Set content length since we know how long the object is
	w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))

	// Decode Range request if present
	code := http.StatusOK
	size := o.Size()
	var options []fs.OpenOption
	if rangeRequest := r.Header.Get("Range"); rangeRequest != "" {
		//fs.Debugf(nil, "Range: request %q", rangeRequest)
		option, err := fs.ParseRangeOption(rangeRequest)
		if err != nil {
			fs.Debugf(remote, "Get request parse range request error: %v", err)
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}
		options = append(options, option)
		offset, limit := option.Decode(o.Size())
		end := o.Size() // exclusive
		if limit >= 0 {
			end = offset + limit
		}
		if end > o.Size() {
			end = o.Size()
		}
		size = end - offset
		// fs.Debugf(nil, "Range: offset=%d, limit=%d, end=%d, size=%d (object size %d)", offset, limit, end, size, o.Size())
		// Content-Range: bytes 0-1023/146515
		w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end-1, o.Size()))
		// fs.Debugf(nil, "Range: Content-Range: %q", w.Header().Get("Content-Range"))
		code = http.StatusPartialContent
	}
	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))

	file, err := o.Open(options...)
	if err != nil {
		fs.Debugf(remote, "Get request open error: %v", err)
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}
	accounting.Stats.Transferring(o.Remote())
	in := accounting.NewAccount(file, o) // account the transfer (no buffering)
	defer func() {
		closeErr := in.Close()
		if closeErr != nil {
			fs.Errorf(remote, "Get request: close failed: %v", closeErr)
			if err == nil {
				err = closeErr
			}
		}
		ok := err == nil
		accounting.Stats.DoneTransferring(o.Remote(), ok)
		if !ok {
			accounting.Stats.Error(err)
		}
	}()

	w.WriteHeader(code)

	n, err := io.Copy(w, in)
	if err != nil {
		fs.Errorf(remote, "Didn't finish writing GET request (wrote %d/%d bytes): %v", n, size, err)
		return
	}
}

// postObject posts an object to the repository
func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote string) {
	if appendOnly {
		// make sure the file does not exist yet
		_, err := s.f.NewObject(remote)
		if err == nil {
			fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode")
			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
			return
		}
	}

	// fs.Debugf(s.f, "content length = %d", r.ContentLength)
	if r.ContentLength >= 0 {
		// Size known use Put
		accounting.Stats.Transferring(remote)
		body := ioutil.NopCloser(r.Body)                                   // we let the server close the body
		in := accounting.NewAccountSizeName(body, r.ContentLength, remote) // account the transfer (no buffering)
		var err error
		defer func() {
			closeErr := in.Close()
			if closeErr != nil {
				fs.Errorf(remote, "Post request: close failed: %v", closeErr)
				if err == nil {
					err = closeErr
				}
			}
			ok := err == nil
			accounting.Stats.DoneTransferring(remote, err == nil)
			if !ok {
				accounting.Stats.Error(err)
			}
		}()
		info := object.NewStaticObjectInfo(remote, time.Now(), r.ContentLength, true, nil, s.f)
		_, err = s.f.Put(in, info)
		if err != nil {
			fs.Errorf(remote, "Post request put error: %v", err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}
	} else {
		// Size unknown use Rcat
		_, err := operations.Rcat(s.f, remote, r.Body, time.Now())
		if err != nil {
			fs.Errorf(remote, "Post request rcat error: %v", err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}
	}
}

// delete the remote
func (s *server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) {
	if appendOnly {
		parts := strings.Split(r.URL.Path, "/")

		// if path doesn't end in "/locks/:name", disallow the operation
		if len(parts) < 2 || parts[len(parts)-2] != "locks" {
			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
			return
		}
	}

	o, err := s.f.NewObject(remote)
	if err != nil {
		fs.Debugf(remote, "Delete request error: %v", err)
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	if err := o.Remove(); err != nil {
		fs.Errorf(remote, "Delete request remove error: %v", err)
		if err == fs.ErrorObjectNotFound {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		} else {
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		}
		return
	}
}

// listItem is an element returned for the restic v2 list response
type listItem struct {
	Name string `json:"name"`
	Size int64  `json:"size"`
}

// return type for list
type listItems []listItem

// add a DirEntry to the listItems
func (ls *listItems) add(entry fs.DirEntry) {
	if o, ok := entry.(fs.Object); ok {
		*ls = append(*ls, listItem{
			Name: path.Base(o.Remote()),
			Size: o.Size(),
		})
	}
}

// listObjects lists all Objects of a given type in an arbitrary order.
func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote string) {
	fs.Debugf(remote, "list request")

	if r.Header.Get("Accept") != resticAPIV2 {
		fs.Errorf(remote, "Restic v2 API required")
		http.Error(w, "Restic v2 API required", http.StatusBadRequest)
		return
	}

	// make sure an empty list is returned, and not a 'nil' value
	ls := listItems{}

	// if remote supports ListR use that directly, otherwise use recursive Walk
	var err error
	if ListR := s.f.Features().ListR; ListR != nil {
		err = ListR(remote, func(entries fs.DirEntries) error {
			for _, entry := range entries {
				ls.add(entry)
			}
			return nil
		})
	} else {
		err = walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error {
			if err == nil {
				for _, entry := range entries {
					ls.add(entry)
				}
			}
			return err
		})
	}

	if err != nil {
		_, err = fserrors.Cause(err)
		if err != fs.ErrorDirNotFound {
			fs.Errorf(remote, "list failed: %#v %T", err, err)
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
	}

	w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2")
	enc := json.NewEncoder(w)
	err = enc.Encode(ls)
	if err != nil {
		fs.Errorf(remote, "failed to write list: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
}

// createRepo creates repository directories.
//
// We don't bother creating the data dirs as rclone will create them on the fly
func (s *server) createRepo(w http.ResponseWriter, r *http.Request, remote string) {
	fs.Infof(remote, "Creating repository")

	if r.URL.Query().Get("create") != "true" {
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	err := s.f.Mkdir(remote)
	if err != nil {
		fs.Errorf(remote, "Create repo failed to Mkdir: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} {
		dirRemote := path.Join(remote, name)
		err := s.f.Mkdir(dirRemote)
		if err != nil {
			fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err)
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
			return
		}
	}
}