rcserver: serve directories as well as files

cmd/rcd/rcd.go

@@ -24,6 +24,8 @@ This is useful if you are controlling rclone via the rc API.
 If you pass in a path to a directory, rclone will serve that directory
 for GET requests on the URL passed in.  It will also open the URL in
 the browser when rclone is run.
+
+See the [rc documentation](/rc/) for more info on the rc flags.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)

cmd/serve/httplib/serve/serve.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"strconv"
 
 	"github.com/ncw/rclone/fs"
@@ -26,6 +27,14 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
 		w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
 	}
 
+	// Set content type
+	mimeType := fs.MimeType(o)
+	if mimeType == "application/octet-stream" && path.Ext(o.Remote()) == "" {
+		// Leave header blank so http server guesses
+	} else {
+		w.Header().Set("Content-Type", mimeType)
+	}
+
 	if r.Method == "HEAD" {
 		return
 	}

docs/content/rc.md

@@ -9,46 +9,78 @@ date: "2018-03-05"
 If rclone is run with the `--rc` flag then it starts an http server
 which can be used to remote control rclone.
 
+If you just want to run a remote control then see the [rcd command](/commands/rclone_rcd/).
+
 **NB** this is experimental and everything here is subject to change!
 
 ## Supported parameters
 
-#### --rc ####
+### --rc
+
 Flag to start the http server listen on remote requests
       
-#### --rc-addr=IP ####
+### --rc-addr=IP
+
 IPaddress:Port or :Port to bind server to. (default "localhost:5572")
 
-#### --rc-cert=KEY ####
+### --rc-cert=KEY
 SSL PEM key (concatenation of certificate and CA certificate)
 
-#### --rc-client-ca=PATH ####
+### --rc-client-ca=PATH
 Client certificate authority to verify clients with
 
-#### --rc-htpasswd=PATH ####
+### --rc-htpasswd=PATH
+
 htpasswd file - if not provided no authentication is done
 
-#### --rc-key=PATH ####
+### --rc-key=PATH
+
 SSL PEM Private key
 
-#### --rc-max-header-bytes=VALUE ####
+### --rc-max-header-bytes=VALUE
+
 Maximum size of request header (default 4096)
 
-#### --rc-user=VALUE ####
+### --rc-user=VALUE
+
 User name for authentication.
 
-#### --rc-pass=VALUE ####
+### --rc-pass=VALUE
+
 Password for authentication.
 
-#### --rc-realm=VALUE ####
+### --rc-realm=VALUE
+
 Realm for authentication (default "rclone")
 
-#### --rc-server-read-timeout=DURATION ####
+### --rc-server-read-timeout=DURATION
+
 Timeout for server reading data (default 1h0m0s)
 
-#### --rc-server-write-timeout=DURATION ####
+### --rc-server-write-timeout=DURATION
+
 Timeout for server writing data (default 1h0m0s)
 
+### --rc-serve
+
+Enable the serving of remote objects via the HTTP interface.  This
+means objects will be accessible at http://127.0.0.1:5572/ by default,
+so you can browse to http://127.0.0.1:5572/ or http://127.0.0.1:5572/*
+to see a listing of the remotes.  Objects may be requested from
+remotes using this syntax http://127.0.0.1:5572/[remote:path]/path/to/object
+
+Default Off.
+
+### --rc-files /path/to/directory
+
+Path to local files to serve on the HTTP server.
+
+If this is set then rclone will serve the files in that directory.  It
+will also open the root in the web browser if specified.  This is for
+implementing browser based GUIs for rclone functions.
+
+Default Off.
+
 ## Accessing the remote control via the rclone rc command
 
 Rclone itself implements the remote control protocol in its `rclone
@@ -394,7 +426,7 @@ The response to a preflight OPTIONS request will echo the requested "Access-Cont
 ### Using POST with URL parameters only
 
 ```
-curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2'
+curl -X POST 'http://localhost:5572/rc/noop?potato=1&sausage=2'
 ```
 
 Response
@@ -409,7 +441,7 @@ Response
 Here is what an error response looks like:
 
 ```
-curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+curl -X POST 'http://localhost:5572/rc/error?potato=1&sausage=2'
 ```
 
 ```
@@ -425,7 +457,7 @@ curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
 Note that curl doesn't return errors to the shell unless you use the `-f` option
 
 ```
-$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
+$ curl -f -X POST 'http://localhost:5572/rc/error?potato=1&sausage=2'
 curl: (22) The requested URL returned error: 400 Bad Request
 $ echo $?
 22
@@ -434,7 +466,7 @@ $ echo $?
 ### Using POST with a form
 
 ```
-curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/
+curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop
 ```
 
 Response
@@ -450,7 +482,7 @@ Note that you can combine these with URL parameters too with the POST
 parameters taking precedence.
 
 ```
-curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4"
+curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop?rutabaga=3&sausage=4"
 ```
 
 Response
@@ -467,7 +499,7 @@ Response
 ### Using POST with a JSON blob
 
 ```
-curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/
+curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop
 ```
 
 response
@@ -483,7 +515,7 @@ This can be combined with URL parameters too if required.  The JSON
 blob takes precedence.
 
 ```
-curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4'
+curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop?rutabaga=3&potato=4'
 ```
 
 ```

fs/rc/rc.go

@@ -19,7 +19,8 @@ import (
 type Options struct {
 	HTTPOptions httplib.Options
 	Enabled     bool   // set to enable the server
-	Files       string // set to enable serving files
+	Serve       bool   // set to serve files from remotes
+	Files       string // set to enable serving files locally
 }
 
 // DefaultOpt is the default values used for Options

fs/rc/rcflags/rcflags.go

@@ -17,6 +17,7 @@ var (
 func AddFlags(flagSet *pflag.FlagSet) {
 	rc.AddOption("rc", &Opt)
 	flags.BoolVarP(flagSet, &Opt.Enabled, "rc", "", false, "Enable the remote control server.")
-	flags.StringVarP(flagSet, &Opt.Files, "rc-files", "", "", "Serve these files on the HTTP server.")
+	flags.StringVarP(flagSet, &Opt.Files, "rc-files", "", "", "Path to local files to serve on the HTTP server.")
+	flags.BoolVarP(flagSet, &Opt.Serve, "rc-serve", "", false, "Enable the serving of remote objects.")
 	httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions)
 }

fs/rc/rcserver/rcserver.go

@@ -5,11 +5,16 @@ import (
 	"encoding/json"
 	"mime"
 	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/ncw/rclone/cmd/serve/httplib"
 	"github.com/ncw/rclone/cmd/serve/httplib/serve"
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
+	"github.com/ncw/rclone/fs/list"
 	"github.com/ncw/rclone/fs/rc"
 	"github.com/pkg/errors"
 	"github.com/skratchdot/open-golang/open"
@@ -18,7 +23,8 @@ import (
 // Start the remote control server if configured
 func Start(opt *rc.Options) {
 	if opt.Enabled {
-		s := newServer(opt)
+		// Serve on the DefaultServeMux so can have global registrations appear
+		s := newServer(opt, http.DefaultServeMux)
 		go s.serve()
 	}
 }
@@ -27,13 +33,13 @@ func Start(opt *rc.Options) {
 type server struct {
 	srv   *httplib.Server
 	files http.Handler
+	opt   *rc.Options
 }
 
-func newServer(opt *rc.Options) *server {
+func newServer(opt *rc.Options, mux *http.ServeMux) *server {
-	// Serve on the DefaultServeMux so can have global registrations appear
-	mux := http.DefaultServeMux
 	s := &server{
 		srv: httplib.NewServer(mux, &opt.HTTPOptions),
+		opt: opt,
 	}
 	mux.HandleFunc("/", s.handler)
 
@@ -89,7 +95,7 @@ func writeError(path string, in rc.Params, w http.ResponseWriter, err error, sta
 
 // handler reads incoming requests and dispatches them
 func (s *server) handler(w http.ResponseWriter, r *http.Request) {
-	path := strings.Trim(r.URL.Path, "/")
+	path := strings.TrimLeft(r.URL.Path, "/")
 
 	w.Header().Add("Access-Control-Allow-Origin", "*")
 
@@ -102,7 +108,7 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
 		s.handlePost(w, r, path)
 	case "OPTIONS":
 		s.handleOptions(w, r, path)
-	case "GET":
+	case "GET", "HEAD":
 		s.handleGet(w, r, path)
 	default:
 		writeError(path, nil, w, errors.Errorf("method %q not allowed", r.Method), http.StatusMethodNotAllowed)
@@ -111,23 +117,29 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *server) handlePost(w http.ResponseWriter, r *http.Request, path string) {
-	// Parse the POST and URL parameters into r.Form, for others r.Form will be empty value
+	contentType := r.Header.Get("Content-Type")
-	err := r.ParseForm()
+
-	if err != nil {
+	values := r.URL.Query()
-		writeError(path, nil, w, errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
+	if contentType == "application/x-www-form-urlencoded" {
-		return
+		// Parse the POST and URL parameters into r.Form, for others r.Form will be empty value
+		err := r.ParseForm()
+		if err != nil {
+			writeError(path, nil, w, errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
+			return
+		}
+		values = r.Form
 	}
 
 	// Read the POST and URL parameters into in
 	in := make(rc.Params)
-	for k, vs := range r.Form {
+	for k, vs := range values {
 		if len(vs) > 0 {
 			in[k] = vs[len(vs)-1]
 		}
 	}
 
 	// Parse a JSON blob from the input
-	if r.Header.Get("Content-Type") == "application/json" {
+	if contentType == "application/json" {
 		err := json.NewDecoder(r.Body).Decode(&in)
 		if err != nil {
 			writeError(path, in, w, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
@@ -138,7 +150,7 @@ func (s *server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 	// Find the call
 	call := rc.Calls.Get(path)
 	if call == nil {
-		writeError(path, in, w, errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
+		writeError(path, in, w, errors.Errorf("couldn't find method %q", path), http.StatusNotFound)
 		return
 	}
 
@@ -176,24 +188,72 @@ func (s *server) handleOptions(w http.ResponseWriter, r *http.Request, path stri
 	w.WriteHeader(http.StatusOK)
 }
 
-func (s *server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
+func (s *server) serveRoot(w http.ResponseWriter, r *http.Request) {
-	// if we have an &fs parameter we are serving from a different fs
+	remotes := config.FileSections()
-	fsName := r.URL.Query().Get("fs")
+	sort.Strings(remotes)
-	if fsName != "" {
+	directory := serve.NewDirectory("")
-		f, err := rc.GetCachedFs(fsName)
+	directory.Title = "List of all rclone remotes."
+	q := url.Values{}
+	for _, remote := range remotes {
+		q.Set("fs", remote)
+		directory.AddEntry("["+remote+":]", true)
+	}
+	directory.Serve(w, r)
+}
+
+func (s *server) serveRemote(w http.ResponseWriter, r *http.Request, path string, fsName string) {
+	f, err := rc.GetCachedFs(fsName)
+	if err != nil {
+		writeError(path, nil, w, errors.Wrap(err, "failed to make Fs"), http.StatusInternalServerError)
+		return
+	}
+	if path == "" || strings.HasSuffix(path, "/") {
+		path = strings.Trim(path, "/")
+		entries, err := list.DirSorted(f, false, path)
 		if err != nil {
-			writeError(path, nil, w, errors.Wrap(err, "failed to make Fs"), http.StatusInternalServerError)
+			writeError(path, nil, w, errors.Wrap(err, "failed to list directory"), http.StatusInternalServerError)
 			return
 		}
+		// Make the entries for display
+		directory := serve.NewDirectory(path)
+		for _, entry := range entries {
+			_, isDir := entry.(fs.Directory)
+			directory.AddEntry(entry.Remote(), isDir)
+		}
+		directory.Serve(w, r)
+	} else {
 		o, err := f.NewObject(path)
 		if err != nil {
 			writeError(path, nil, w, errors.Wrap(err, "failed to find object"), http.StatusInternalServerError)
 			return
 		}
 		serve.Object(w, r, o)
-	} else if s.files == nil {
-		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
-	} else {
-		s.files.ServeHTTP(w, r)
 	}
 }
+
+// Match URLS of the form [fs]/remote
+var fsMatch = regexp.MustCompile(`^\[(.*?)\](.*)$`)
+
+func (s *server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
+	// Look to see if this has an fs in the path
+	match := fsMatch.FindStringSubmatch(path)
+	switch {
+	case match != nil && s.opt.Serve:
+		// Serve /[fs]/remote files
+		s.serveRemote(w, r, match[2], match[1])
+		return
+	case path == "*" && s.opt.Serve:
+		// Serve /* as the remote listing
+		s.serveRoot(w, r)
+		return
+	case s.files != nil:
+		// Serve the files
+		s.files.ServeHTTP(w, r)
+		return
+	case path == "" && s.opt.Serve:
+		// Serve the root as a remote listing
+		s.serveRoot(w, r)
+		return
+	}
+	http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+}

fs/rc/rcserver/rcserver_test.go

@@ -0,0 +1,570 @@
+// +build go1.8
+
+package rcserver
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+	"time"
+
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testBindAddress = "localhost:51781"
+	testURL         = "http://" + testBindAddress + "/"
+	testFs          = "testdata/files"
+	remoteURL       = "[" + testFs + "]/" // initial URL path to fetch from that remote
+)
+
+// Test the RC server runs and we can do HTTP fetches from it.
+// We'll do the majority of the testing with the httptest framework
+func TestRcServer(t *testing.T) {
+	opt := rc.DefaultOpt
+	opt.HTTPOptions.ListenAddr = testBindAddress
+	opt.Enabled = true
+	opt.Serve = true
+	opt.Files = testFs
+	mux := http.NewServeMux()
+	rcServer := newServer(&opt, mux)
+	go rcServer.serve()
+	defer rcServer.srv.Close()
+
+	// Do the simplest possible test to check the server is alive
+	// Do it a few times to wait for the server to start
+	var resp *http.Response
+	var err error
+	for i := 0; i < 10; i++ {
+		resp, err = http.Get(testURL + "file.txt")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	require.NoError(t, err)
+	body, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "this is file1.txt\n", string(body))
+}
+
+type testRun struct {
+	Name        string
+	URL         string
+	Status      int
+	Method      string
+	Range       string
+	Body        string
+	ContentType string
+	Expected    string
+	Contains    *regexp.Regexp
+	Headers     map[string]string
+}
+
+// Run a suite of tests
+func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
+	mux := http.NewServeMux()
+	rcServer := newServer(opt, mux)
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			method := test.Method
+			if method == "" {
+				method = "GET"
+			}
+			var inBody io.Reader
+			if test.Body != "" {
+				buf := bytes.NewBufferString(test.Body)
+				inBody = buf
+			}
+			req, err := http.NewRequest(method, "http://1.2.3.4/"+test.URL, inBody)
+			require.NoError(t, err)
+			if test.Range != "" {
+				req.Header.Add("Range", test.Range)
+			}
+			if test.ContentType != "" {
+				req.Header.Add("Content-Type", test.ContentType)
+			}
+
+			w := httptest.NewRecorder()
+			rcServer.handler(w, req)
+			resp := w.Result()
+
+			assert.Equal(t, test.Status, resp.StatusCode)
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+
+			if test.Contains == nil {
+				assert.Equal(t, test.Expected, string(body))
+			} else {
+				assert.True(t, test.Contains.Match(body), fmt.Sprintf("body didn't match: %v: %v", test.Contains, string(body)))
+			}
+
+			for k, v := range test.Headers {
+				assert.Equal(t, v, resp.Header.Get(k), k)
+			}
+		})
+	}
+}
+
+// return an enabled rc
+func newTestOpt() rc.Options {
+	opt := rc.DefaultOpt
+	opt.Enabled = true
+	return opt
+}
+
+func TestFileServing(t *testing.T) {
+	tests := []testRun{{
+		Name:   "index",
+		URL:    "",
+		Status: http.StatusOK,
+		Expected: `<pre>
+<a href="dir/">dir/</a>
+<a href="file.txt">file.txt</a>
+</pre>
+`,
+	}, {
+		Name:     "notfound",
+		URL:      "notfound",
+		Status:   http.StatusNotFound,
+		Expected: "404 page not found\n",
+	}, {
+		Name:     "dirnotfound",
+		URL:      "dirnotfound/",
+		Status:   http.StatusNotFound,
+		Expected: "404 page not found\n",
+	}, {
+		Name:   "dir",
+		URL:    "dir/",
+		Status: http.StatusOK,
+		Expected: `<pre>
+<a href="file2.txt">file2.txt</a>
+</pre>
+`,
+	}, {
+		Name:     "file",
+		URL:      "file.txt",
+		Status:   http.StatusOK,
+		Expected: "this is file1.txt\n",
+		Headers: map[string]string{
+			"Content-Length": "18",
+		},
+	}, {
+		Name:     "file2",
+		URL:      "dir/file2.txt",
+		Status:   http.StatusOK,
+		Expected: "this is dir/file2.txt\n",
+	}, {
+		Name:     "file-head",
+		URL:      "file.txt",
+		Method:   "HEAD",
+		Status:   http.StatusOK,
+		Expected: ``,
+		Headers: map[string]string{
+			"Content-Length": "18",
+		},
+	}, {
+		Name:     "file-range",
+		URL:      "file.txt",
+		Status:   http.StatusPartialContent,
+		Range:    "bytes=8-12",
+		Expected: `file1`,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+func TestRemoteServing(t *testing.T) {
+	tests := []testRun{
+		// Test serving files from the test remote
+		{
+			Name:   "index",
+			URL:    remoteURL + "",
+			Status: http.StatusOK,
+			Expected: `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Directory listing of /</title>
+</head>
+<body>
+<h1>Directory listing of /</h1>
+<a href="dir/">dir/</a><br />
+<a href="file.txt">file.txt</a><br />
+</body>
+</html>
+`,
+		}, {
+			Name:   "notfound-index",
+			URL:    "[notfound]/",
+			Status: http.StatusNotFound,
+			Expected: `{
+	"error": "failed to list directory: directory not found",
+	"input": null,
+	"path": "",
+	"status": 404
+}
+`,
+		}, {
+			Name:   "notfound",
+			URL:    remoteURL + "notfound",
+			Status: http.StatusNotFound,
+			Expected: `{
+	"error": "failed to find object: object not found",
+	"input": null,
+	"path": "/notfound",
+	"status": 404
+}
+`,
+		}, {
+			Name:   "dirnotfound",
+			URL:    remoteURL + "dirnotfound/",
+			Status: http.StatusNotFound,
+			Expected: `{
+	"error": "failed to list directory: directory not found",
+	"input": null,
+	"path": "dirnotfound",
+	"status": 404
+}
+`,
+		}, {
+			Name:   "dir",
+			URL:    remoteURL + "dir/",
+			Status: http.StatusOK,
+			Expected: `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Directory listing of /dir</title>
+</head>
+<body>
+<h1>Directory listing of /dir</h1>
+<a href="file2.txt">file2.txt</a><br />
+</body>
+</html>
+`,
+		}, {
+			Name:     "file",
+			URL:      remoteURL + "file.txt",
+			Status:   http.StatusOK,
+			Expected: "this is file1.txt\n",
+			Headers: map[string]string{
+				"Content-Length": "18",
+			},
+		}, {
+			Name:     "file2",
+			URL:      remoteURL + "dir/file2.txt",
+			Status:   http.StatusOK,
+			Expected: "this is dir/file2.txt\n",
+		}, {
+			Name:     "file-head",
+			URL:      remoteURL + "file.txt",
+			Method:   "HEAD",
+			Status:   http.StatusOK,
+			Expected: ``,
+			Headers: map[string]string{
+				"Content-Length": "18",
+			},
+		}, {
+			Name:     "file-range",
+			URL:      remoteURL + "file.txt",
+			Status:   http.StatusPartialContent,
+			Range:    "bytes=8-12",
+			Expected: `file1`,
+		}, {
+			Name:   "bad-remote",
+			URL:    "[notfoundremote:]/",
+			Status: http.StatusInternalServerError,
+			Expected: `{
+	"error": "failed to make Fs: didn't find section in config file",
+	"input": null,
+	"path": "/",
+	"status": 500
+}
+`,
+		}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+func TestRC(t *testing.T) {
+	tests := []testRun{{
+		Name:   "rc-root",
+		URL:    "",
+		Method: "POST",
+		Status: http.StatusNotFound,
+		Expected: `{
+	"error": "couldn't find method \"\"",
+	"input": {},
+	"path": "",
+	"status": 404
+}
+`,
+	}, {
+		Name:     "rc-noop",
+		URL:      "rc/noop",
+		Method:   "POST",
+		Status:   http.StatusOK,
+		Expected: "{}\n",
+	}, {
+		Name:   "rc-error",
+		URL:    "rc/error",
+		Method: "POST",
+		Status: http.StatusInternalServerError,
+		Expected: `{
+	"error": "arbitrary error on input map[]",
+	"input": {},
+	"path": "rc/error",
+	"status": 500
+}
+`,
+	}, {
+		Name:     "core-gc",
+		URL:      "core/gc", // returns nil, nil so check it is made into {}
+		Method:   "POST",
+		Status:   http.StatusOK,
+		Expected: "{}\n",
+	}, {
+		Name:   "url-params",
+		URL:    "rc/noop?param1=potato&param2=sausage",
+		Method: "POST",
+		Status: http.StatusOK,
+		Expected: `{
+	"param1": "potato",
+	"param2": "sausage"
+}
+`,
+	}, {
+		Name:        "json",
+		URL:         "rc/noop",
+		Method:      "POST",
+		Body:        `{ "param1":"string", "param2":true }`,
+		ContentType: "application/json",
+		Status:      http.StatusOK,
+		Expected: `{
+	"param1": "string",
+	"param2": true
+}
+`,
+	}, {
+		Name:        "json-and-url-params",
+		URL:         "rc/noop?param1=potato&param2=sausage",
+		Method:      "POST",
+		Body:        `{ "param1":"string", "param3":true }`,
+		ContentType: "application/json",
+		Status:      http.StatusOK,
+		Expected: `{
+	"param1": "string",
+	"param2": "sausage",
+	"param3": true
+}
+`,
+	}, {
+		Name:        "json-bad",
+		URL:         "rc/noop?param1=potato&param2=sausage",
+		Method:      "POST",
+		Body:        `{ param1":"string", "param3":true }`,
+		ContentType: "application/json",
+		Status:      http.StatusBadRequest,
+		Expected: `{
+	"error": "failed to read input JSON: invalid character 'p' looking for beginning of object key string",
+	"input": {
+		"param1": "potato",
+		"param2": "sausage"
+	},
+	"path": "rc/noop",
+	"status": 400
+}
+`,
+	}, {
+		Name:        "form",
+		URL:         "rc/noop",
+		Method:      "POST",
+		Body:        `param1=string&param2=true`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusOK,
+		Expected: `{
+	"param1": "string",
+	"param2": "true"
+}
+`,
+	}, {
+		Name:        "form-and-url-params",
+		URL:         "rc/noop?param1=potato&param2=sausage",
+		Method:      "POST",
+		Body:        `param1=string&param3=true`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusOK,
+		Expected: `{
+	"param1": "potato",
+	"param2": "sausage",
+	"param3": "true"
+}
+`,
+	}, {
+		Name:        "form-bad",
+		URL:         "rc/noop?param1=potato&param2=sausage",
+		Method:      "POST",
+		Body:        `%zz`,
+		ContentType: "application/x-www-form-urlencoded",
+		Status:      http.StatusBadRequest,
+		Expected: `{
+	"error": "failed to parse form/URL parameters: invalid URL escape \"%zz\"",
+	"input": null,
+	"path": "rc/noop",
+	"status": 400
+}
+`,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+func TestMethods(t *testing.T) {
+	tests := []testRun{{
+		Name:     "options",
+		URL:      "",
+		Method:   "OPTIONS",
+		Status:   http.StatusOK,
+		Expected: "",
+		Headers: map[string]string{
+			"Access-Control-Allow-Origin":  "*",
+			"Access-Control-Allow-Headers": "",
+		},
+	}, {
+		Name:   "bad",
+		URL:    "",
+		Method: "POTATO",
+		Status: http.StatusMethodNotAllowed,
+		Expected: `{
+	"error": "method \"POTATO\" not allowed",
+	"input": null,
+	"path": "",
+	"status": 405
+}
+`,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+var matchRemoteDirListing = regexp.MustCompile(`<title>List of all rclone remotes.</title>`)
+
+func TestServingRoot(t *testing.T) {
+	tests := []testRun{{
+		Name:     "rootlist",
+		URL:      "*",
+		Status:   http.StatusOK,
+		Contains: matchRemoteDirListing,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+func TestServingRootNoFiles(t *testing.T) {
+	tests := []testRun{{
+		Name:     "rootlist",
+		URL:      "",
+		Status:   http.StatusOK,
+		Contains: matchRemoteDirListing,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = ""
+	testServer(t, tests, &opt)
+}
+
+func TestNoFiles(t *testing.T) {
+	tests := []testRun{{
+		Name:     "file",
+		URL:      "file.txt",
+		Status:   http.StatusNotFound,
+		Expected: "Not Found\n",
+	}, {
+		Name:     "dir",
+		URL:      "dir/",
+		Status:   http.StatusNotFound,
+		Expected: "Not Found\n",
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = ""
+	testServer(t, tests, &opt)
+}
+
+func TestNoServe(t *testing.T) {
+	tests := []testRun{{
+		Name:     "file",
+		URL:      remoteURL + "file.txt",
+		Status:   http.StatusNotFound,
+		Expected: "404 page not found\n",
+	}, {
+		Name:     "dir",
+		URL:      remoteURL + "dir/",
+		Status:   http.StatusNotFound,
+		Expected: "404 page not found\n",
+	}}
+	opt := newTestOpt()
+	opt.Serve = false
+	opt.Files = testFs
+	testServer(t, tests, &opt)
+}
+
+func TestRCAsync(t *testing.T) {
+	tests := []testRun{{
+		Name:        "ok",
+		URL:         "rc/noop",
+		Method:      "POST",
+		ContentType: "application/json",
+		Body:        `{ "_async":true }`,
+		Status:      http.StatusOK,
+		Expected: `{
+	"jobid": 1
+}
+`,
+	}, {
+		Name:        "bad",
+		URL:         "rc/noop",
+		Method:      "POST",
+		ContentType: "application/json",
+		Body:        `{ "_async":"truthy" }`,
+		Status:      http.StatusBadRequest,
+		Expected: `{
+	"error": "couldn't parse key \"_async\" (truthy) as bool: strconv.ParseBool: parsing \"truthy\": invalid syntax",
+	"input": {
+		"_async": "truthy"
+	},
+	"path": "rc/noop",
+	"status": 400
+}
+`,
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = ""
+	testServer(t, tests, &opt)
+}

fs/rc/rcserver/testdata/files/dir/file2.txt

@@ -0,0 +1 @@
+this is dir/file2.txt

fs/rc/rcserver/testdata/files/file.txt

@@ -0,0 +1 @@
+this is file1.txt