github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-21 22:50:16 +01:00
Code Releases Activity

backend: add new backend command for backend specific commands

Browse Source 
These commands are for implementing backend specific
functionality. They have documentation which is placed automatically
into the backend doc.

There is a simple test for the feature in the backend tests.
This commit is contained in:
Nick Craig-Wood 2020-04-28 12:58:34 +01:00
parent 195d152785
commit 1aa1a2c174
5 changed files with 242 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bin/make_backend_docs.py
View File

@ -4,6 +4,7 @@ Make backend documentation
"""
import os
import io
import subprocess
marker = "<!--- autogenerated options"
@ -19,6 +20,11 @@ def output_docs(backend, out):
out.flush()
subprocess.check_call(["rclone", "help", "backend", backend], stdout=out)
def output_backend_tool_docs(backend, out):
"""Output documentation for backend tool to out"""
out.flush()
subprocess.call(["rclone", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL)
def alter_doc(backend):
"""Alter the documentation for backend"""
doc_file = "docs/content/"+backend+".md"
@ -35,6 +41,7 @@ def alter_doc(backend):
start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
out_file.write(start_full)
output_docs(backend, out_file)
output_backend_tool_docs(backend, out_file)
out_file.write(stop+" -->\n")
altered = True
if not in_docs:

1
cmd/all/all.go
View File

@ -6,6 +6,7 @@ import (
_ "github.com/rclone/rclone/cmd"
_ "github.com/rclone/rclone/cmd/about"
_ "github.com/rclone/rclone/cmd/authorize"
_ "github.com/rclone/rclone/cmd/backend"
_ "github.com/rclone/rclone/cmd/cachestats"
_ "github.com/rclone/rclone/cmd/cat"
_ "github.com/rclone/rclone/cmd/check"

169
cmd/backend/backend.go Normal file
View File

@ -0,0 +1,169 @@
package backend
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"github.com/pkg/errors"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/rc"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
var (
options []string
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.StringArrayVarP(cmdFlags, &options, "option", "o", options, "Option in the form name=value or name.")
}
var commandDefinition = &cobra.Command{
Use: "backend <command> remote:path [opts] <args>",
Short: `Run a backend specific command.`,
Long: `
This runs a backend specific command. The commands themselves (except
for "help" and "features") are defined by the backends and you should
see the backend docs for definitions.
You can discover what commands a backend implements by using
rclone backend help remote:
rclone backend help <backendname>
You can also discover information about the backend using (see
[operations/fsinfo](/rc/#operations/fsinfo) in the remote control docs
for more info).
rclone backend features remote:
Pass options to the backend command with -o. This should be key=value or key, eg:
rclone backend stats remote:path stats -o format=json -o long
Pass arguments to the backend by placing them on the end of the line
rclone backend cleanup remote:path file1 file2 file3
Note to run these commands on a running backend then see
[backend/command](/rc/#backend/command) in the rc docs.
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(2, 1E6, command, args)
name, remote := args[0], args[1]
cmd.Run(false, false, command, func() error {
// show help if remote is a backend name
if name == "help" {
fsInfo, err := fs.Find(remote)
if err == nil {
return showHelp(fsInfo)
}
}
// Create remote
fsInfo, configName, fsPath, config, err := fs.ConfigFs(remote)
if err != nil {
return err
}
f, err := fsInfo.NewFs(configName, fsPath, config)
if err != nil {
return err
}
// Run the command
var out interface{}
switch name {
case "help":
return showHelp(fsInfo)
case "features":
out = operations.GetFsInfo(f)
default:
doCommand := f.Features().Command
if doCommand == nil {
return errors.Errorf("%v: doesn't support backend commands", f)
}
arg := args[2:]
opt := rc.ParseOptions(options)
out, err = doCommand(context.Background(), name, arg, opt)
}
if err != nil {
return errors.Wrapf(err, "command %q failed", name)
}
// Output the result
switch x := out.(type) {
case nil:
case string:
fmt.Println(out)
case []string:
for line := range x {
fmt.Println(line)
}
default:
// Write indented JSON to the output
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", "\t")
err = enc.Encode(out)
if err != nil {
return errors.Wrap(err, "failed to write JSON")
}
}
return nil
})
return nil
},
}
// show help for a backend
func showHelp(fsInfo *fs.RegInfo) error {
cmds := fsInfo.CommandHelp
name := fsInfo.Name
if len(cmds) == 0 {
return errors.Errorf("%s backend has no commands", name)
}
fmt.Printf("### Backend commands\n\n")
fmt.Printf(`Here are the commands specific to the %s backend.
Run them with with
rclone backend COMMAND remote:
The help below will explain what arguments each command takes.
See [the "rclone backend" command](/commands/rclone_backend/) for more
info on how to pass options and arguments.
These can be run on a running backend using the rc command
[backend/command](/rc/#backend/command).
`, name)
for _, cmd := range cmds {
fmt.Printf("#### %s\n\n", cmd.Name)
fmt.Printf("%s\n\n", cmd.Short)
fmt.Printf(" rclone backend %s remote: [options] [<arguments>+]\n\n", cmd.Name)
if cmd.Long != "" {
fmt.Printf("%s\n\n", cmd.Long)
}
if len(cmd.Opts) != 0 {
fmt.Printf("Options:\n\n")
ks := []string{}
for k := range cmd.Opts {
ks = append(ks, k)
}
sort.Strings(ks)
for _, k := range ks {
v := cmd.Opts[k]
fmt.Printf("- %q: %s\n", k, v)
}
fmt.Printf("\n")
}
}
return nil
}

42
fs/fs.go
View File

@ -70,6 +70,7 @@ var (
ErrorPermissionDenied = errors.New("permission denied")
ErrorCantShareDirectories = errors.New("this backend can't share directories with link")
ErrorNotImplemented = errors.New("optional feature not implemented")
ErrorCommandNotFound = errors.New("command not found")
)
// RegInfo provides information about a filesystem
@ -88,6 +89,8 @@ type RegInfo struct {
Config func(name string, config configmap.Mapper) `json:"-"`
// Options for the Fs configuration
Options Options
// The command help, if any
CommandHelp []CommandHelp
}
// FileName returns the on disk file name for this backend
@ -634,6 +637,17 @@ type Features struct {
// Disconnect the current user
Disconnect func(ctx context.Context) error
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}
// Disable nil's out the named feature. If it isn't found then it
@ -755,6 +769,9 @@ func (ft *Features) Fill(f Fs) *Features {
if do, ok := f.(Disconnecter); ok {
ft.Disconnect = do.Disconnect
}
if do, ok := f.(Commander); ok {
ft.Command = do.Command
}
return ft.DisableList(Config.DisableFeatures)
}
@ -830,6 +847,7 @@ func (ft *Features) Mask(f Fs) *Features {
if mask.Disconnect == nil {
ft.Disconnect = nil
}
// Command is always local so we don't mask it
return ft.DisableList(Config.DisableFeatures)
}
@ -1051,6 +1069,30 @@ type Disconnecter interface {
Disconnect(ctx context.Context) error
}
// CommandHelp describes a single backend Command
//
// These are automatically inserted in the docs
type CommandHelp struct {
Name string // Name of the command, eg "link"
Short string // Single line description
Long string // Long multi-line description
Opts map[string]string // maps option name to a single line help
}
// Commander is an iterface to wrap the Command function
type Commander interface {
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}
// ObjectsChan is a channel of Objects
type ObjectsChan chan Object

26
fstest/fstests/fstests.go
View File

@ -292,9 +292,10 @@ func Run(t *testing.T, opt *Opt) {
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
}
isLocalRemote bool
purged bool // whether the dir has been purged or not
ctx = context.Background()
isLocalRemote bool
purged bool // whether the dir has been purged or not
ctx = context.Background()
unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
)
if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
@ -398,6 +399,9 @@ func Run(t *testing.T, opt *Opt) {
if stringsContains(vName, opt.UnimplementableFsMethods) {
continue
}
if stringsContains(vName, unwrappableFsMethods) {
continue
}
field := v.Field(i)
// skip the bools
if field.Type().Kind() == reflect.Bool {
@ -409,6 +413,22 @@ func Run(t *testing.T, opt *Opt) {
}
})
// Check to see if Fs advertises commands and they work and have docs
t.Run("FsCommand", func(t *testing.T) {
skipIfNotOk(t)
doCommand := remote.Features().Command
if doCommand == nil {
t.Skip("No commands in this remote")
}
// Check the correct error is generated
_, err := doCommand(context.Background(), "NOTFOUND", nil, nil)
assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found")
// Check there are some commands in the fsInfo
fsInfo, _, _, _, err := fs.ConfigFs(remoteName)
require.NoError(t, err)
assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp")
})
// TestFsRmdirNotFound tests deleting a non existent directory
t.Run("FsRmdirNotFound", func(t *testing.T) {
skipIfNotOk(t)