2016-08-04 23:18:56 +02:00
|
|
|
// Package cmd implemnts the rclone command
|
|
|
|
//
|
|
|
|
// It is in a sub package so it's internals can be re-used elsewhere
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
// FIXME only attach the remote flags when using a remote???
|
|
|
|
// would probably mean bringing all the flags in to here? Or define some flagsets in fs...
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"log"
|
|
|
|
"os"
|
|
|
|
"path"
|
2016-11-22 05:04:05 +01:00
|
|
|
"regexp"
|
2016-08-04 23:18:56 +02:00
|
|
|
"runtime"
|
|
|
|
"runtime/pprof"
|
2016-10-23 18:34:17 +02:00
|
|
|
"strings"
|
2016-08-04 23:18:56 +02:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"github.com/spf13/pflag"
|
|
|
|
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Flags
|
2016-12-20 16:50:46 +01:00
|
|
|
cpuProfile = fs.StringP("cpuprofile", "", "", "Write cpu profile to file")
|
|
|
|
memProfile = fs.StringP("memprofile", "", "", "Write memory profile to file")
|
|
|
|
statsInterval = fs.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
|
|
|
|
dataRateUnit = fs.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
|
2016-08-04 23:18:56 +02:00
|
|
|
version bool
|
2016-12-20 16:50:46 +01:00
|
|
|
retries = fs.IntP("retries", "", 3, "Retry operations this many times if they fail")
|
2016-08-04 23:18:56 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// Root is the main rclone command
|
|
|
|
var Root = &cobra.Command{
|
|
|
|
Use: "rclone",
|
|
|
|
Short: "Sync files and directories to and from local and remote object stores - " + fs.Version,
|
|
|
|
Long: `
|
|
|
|
Rclone is a command line program to sync files and directories to and
|
|
|
|
from various cloud storage systems, such as:
|
|
|
|
|
|
|
|
* Google Drive
|
|
|
|
* Amazon S3
|
|
|
|
* Openstack Swift / Rackspace cloud files / Memset Memstore
|
|
|
|
* Dropbox
|
|
|
|
* Google Cloud Storage
|
|
|
|
* Amazon Drive
|
|
|
|
* Microsoft One Drive
|
|
|
|
* Hubic
|
|
|
|
* Backblaze B2
|
|
|
|
* Yandex Disk
|
|
|
|
* The local filesystem
|
|
|
|
|
|
|
|
Features
|
|
|
|
|
|
|
|
* MD5/SHA1 hashes checked at all times for file integrity
|
|
|
|
* Timestamps preserved on files
|
|
|
|
* Partial syncs supported on a whole file basis
|
|
|
|
* Copy mode to just copy new/changed files
|
|
|
|
* Sync (one way) mode to make a directory identical
|
|
|
|
* Check mode to check for file hash equality
|
|
|
|
* Can sync to and from network, eg two different cloud accounts
|
|
|
|
|
|
|
|
See the home page for installation, usage, documentation, changelog
|
|
|
|
and configuration walkthroughs.
|
|
|
|
|
|
|
|
* http://rclone.org/
|
|
|
|
`,
|
2017-02-20 17:33:45 +01:00
|
|
|
PersistentPostRun: func(cmd *cobra.Command, args []string) {
|
2017-02-20 17:36:25 +01:00
|
|
|
fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
|
2017-02-20 17:33:45 +01:00
|
|
|
runAtExitFunctions()
|
|
|
|
},
|
2016-08-05 18:12:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// runRoot implements the main rclone command with no subcommands
|
|
|
|
func runRoot(cmd *cobra.Command, args []string) {
|
|
|
|
if version {
|
|
|
|
ShowVersion()
|
|
|
|
os.Exit(0)
|
|
|
|
} else {
|
|
|
|
_ = Root.Usage()
|
|
|
|
fmt.Fprintf(os.Stderr, "Command not found.\n")
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
2016-08-05 18:12:27 +02:00
|
|
|
Root.Run = runRoot
|
2016-08-04 23:18:56 +02:00
|
|
|
Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
|
|
|
|
cobra.OnInitialize(initConfig)
|
|
|
|
}
|
|
|
|
|
2016-08-05 18:12:27 +02:00
|
|
|
// ShowVersion prints the version to stdout
|
|
|
|
func ShowVersion() {
|
2016-08-04 23:18:56 +02:00
|
|
|
fmt.Printf("rclone %s\n", fs.Version)
|
|
|
|
}
|
|
|
|
|
2016-10-23 18:34:17 +02:00
|
|
|
// newFsFile creates a dst Fs from a name but may point to a file.
|
2016-08-04 23:18:56 +02:00
|
|
|
//
|
2016-10-23 18:34:17 +02:00
|
|
|
// It returns a string with the file name if points to a file
|
|
|
|
func newFsFile(remote string) (fs.Fs, string) {
|
2016-08-04 23:18:56 +02:00
|
|
|
fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
|
|
|
}
|
|
|
|
f, err := fsInfo.NewFs(configName, fsPath)
|
2016-10-23 18:34:17 +02:00
|
|
|
switch err {
|
|
|
|
case fs.ErrorIsFile:
|
|
|
|
return f, path.Base(fsPath)
|
|
|
|
case nil:
|
|
|
|
return f, ""
|
|
|
|
default:
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
|
|
|
}
|
|
|
|
return nil, ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// newFsSrc creates a src Fs from a name
|
|
|
|
//
|
|
|
|
// It returns a string with the file name if limiting to one file
|
|
|
|
//
|
|
|
|
// This can point to a file
|
|
|
|
func newFsSrc(remote string) (fs.Fs, string) {
|
|
|
|
f, fileName := newFsFile(remote)
|
|
|
|
if fileName != "" {
|
2016-08-04 23:18:56 +02:00
|
|
|
if !fs.Config.Filter.InActive() {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatalf("Can't limit to single files when using filters: %v", remote)
|
|
|
|
}
|
|
|
|
// Limit transfers to this file
|
2016-10-23 18:34:17 +02:00
|
|
|
err := fs.Config.Filter.AddFile(fileName)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatalf("Failed to limit to single file %q: %v", remote, err)
|
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
// Set --no-traverse as only one file
|
|
|
|
fs.Config.NoTraverse = true
|
|
|
|
}
|
2016-10-23 18:34:17 +02:00
|
|
|
return f, fileName
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// newFsDst creates a dst Fs from a name
|
|
|
|
//
|
|
|
|
// This must point to a directory
|
|
|
|
func newFsDst(remote string) fs.Fs {
|
|
|
|
f, err := fs.NewFs(remote)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
|
|
|
}
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewFsSrcDst creates a new src and dst fs from the arguments
|
|
|
|
func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) {
|
2016-10-23 18:34:17 +02:00
|
|
|
fsrc, _ := newFsSrc(args[0])
|
|
|
|
fdst := newFsDst(args[1])
|
2016-08-04 23:18:56 +02:00
|
|
|
fs.CalculateModifyWindow(fdst, fsrc)
|
2016-08-06 01:07:36 +02:00
|
|
|
return fsrc, fdst
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
|
|
|
|
2016-10-23 18:34:17 +02:00
|
|
|
// RemoteSplit splits a remote into a parent and a leaf
|
|
|
|
//
|
2017-02-22 22:24:04 +01:00
|
|
|
// if it returns leaf as an empty string then remote is a directory
|
|
|
|
//
|
|
|
|
// if it returns parent as an empty string then that means the current directory
|
|
|
|
//
|
|
|
|
// The returned values have the property that parent + leaf == remote
|
2016-10-23 18:34:17 +02:00
|
|
|
func RemoteSplit(remote string) (parent string, leaf string) {
|
|
|
|
// Split remote on :
|
|
|
|
i := strings.Index(remote, ":")
|
|
|
|
remoteName := ""
|
|
|
|
remotePath := remote
|
|
|
|
if i >= 0 {
|
|
|
|
remoteName = remote[:i+1]
|
|
|
|
remotePath = remote[i+1:]
|
2017-02-22 22:24:04 +01:00
|
|
|
} else if strings.HasSuffix(remotePath, "/") {
|
|
|
|
// if no : and ends with / must be directory
|
|
|
|
return remotePath, ""
|
2016-10-23 18:34:17 +02:00
|
|
|
}
|
|
|
|
// Construct new remote name without last segment
|
|
|
|
parent, leaf = path.Split(remotePath)
|
2017-02-22 22:24:04 +01:00
|
|
|
return remoteName + parent, leaf
|
2016-10-23 18:34:17 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewFsSrcDstFiles creates a new src and dst fs from the arguments
|
|
|
|
// If src is a file then srcFileName and dstFileName will be non-empty
|
|
|
|
func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {
|
|
|
|
fsrc, srcFileName = newFsSrc(args[0])
|
|
|
|
// If copying a file...
|
|
|
|
dstRemote := args[1]
|
|
|
|
if srcFileName != "" {
|
|
|
|
dstRemote, dstFileName = RemoteSplit(dstRemote)
|
|
|
|
if dstRemote == "" {
|
2017-02-22 22:24:04 +01:00
|
|
|
dstRemote = "."
|
|
|
|
}
|
|
|
|
if dstFileName == "" {
|
|
|
|
log.Fatalf("%q is a directory", args[1])
|
2016-10-23 18:34:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
fdst = newFsDst(dstRemote)
|
|
|
|
fs.CalculateModifyWindow(fdst, fsrc)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-08-04 23:18:56 +02:00
|
|
|
// NewFsSrc creates a new src fs from the arguments
|
|
|
|
func NewFsSrc(args []string) fs.Fs {
|
2016-10-23 18:34:17 +02:00
|
|
|
fsrc, _ := newFsSrc(args[0])
|
2016-08-04 23:18:56 +02:00
|
|
|
fs.CalculateModifyWindow(fsrc)
|
|
|
|
return fsrc
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewFsDst creates a new dst fs from the arguments
|
|
|
|
//
|
|
|
|
// Dst fs-es can't point to single files
|
|
|
|
func NewFsDst(args []string) fs.Fs {
|
|
|
|
fdst := newFsDst(args[0])
|
|
|
|
fs.CalculateModifyWindow(fdst)
|
|
|
|
return fdst
|
|
|
|
}
|
|
|
|
|
2016-12-15 18:40:17 +01:00
|
|
|
// ShowStats returns true if the user added a `--stats` flag to the command line.
|
|
|
|
//
|
|
|
|
// This is called by Run to override the default value of the
|
|
|
|
// showStats passed in.
|
|
|
|
func ShowStats() bool {
|
|
|
|
statsIntervalFlag := pflag.Lookup("stats")
|
|
|
|
return statsIntervalFlag != nil && statsIntervalFlag.Changed
|
|
|
|
}
|
|
|
|
|
2016-08-04 23:18:56 +02:00
|
|
|
// Run the function with stats and retries if required
|
2016-12-04 17:52:24 +01:00
|
|
|
func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
|
2016-08-04 23:18:56 +02:00
|
|
|
var err error
|
2016-12-04 17:52:24 +01:00
|
|
|
var stopStats chan struct{}
|
2016-12-15 18:40:17 +01:00
|
|
|
if !showStats && ShowStats() {
|
|
|
|
showStats = true
|
|
|
|
}
|
2016-12-04 17:52:24 +01:00
|
|
|
if showStats {
|
2016-12-01 09:49:47 +01:00
|
|
|
stopStats = StartStats()
|
2016-12-04 17:52:24 +01:00
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
for try := 1; try <= *retries; try++ {
|
|
|
|
err = f()
|
|
|
|
if !Retry || (err == nil && !fs.Stats.Errored()) {
|
2016-09-12 16:42:57 +02:00
|
|
|
if try > 1 {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries)
|
2016-09-12 16:42:57 +02:00
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if fs.IsFatalError(err) {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Fatal error received - not attempting retries")
|
2016-08-04 23:18:56 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if fs.IsNoRetryError(err) {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Can't retry this error - not attempting retries")
|
2016-08-04 23:18:56 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if err != nil {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, fs.Stats.GetErrors(), err)
|
2016-08-04 23:18:56 +02:00
|
|
|
} else {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, fs.Stats.GetErrors())
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
|
|
|
if try < *retries {
|
|
|
|
fs.Stats.ResetErrors()
|
|
|
|
}
|
|
|
|
}
|
2016-12-04 17:52:24 +01:00
|
|
|
if showStats {
|
|
|
|
close(stopStats)
|
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Failed to %s: %v", cmd.Name(), err)
|
|
|
|
}
|
2017-02-09 22:22:46 +01:00
|
|
|
if showStats && (fs.Stats.Errored() || *statsInterval > 0) {
|
2017-02-09 18:08:51 +01:00
|
|
|
fs.Infof(nil, "%s", fs.Stats)
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
2017-02-09 22:22:46 +01:00
|
|
|
fs.Debugf(nil, "Go routines at exit %d\n", runtime.NumGoroutine())
|
2016-08-04 23:18:56 +02:00
|
|
|
if fs.Stats.Errored() {
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// CheckArgs checks there are enough arguments and prints a message if not
|
|
|
|
func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
|
|
|
|
if len(args) < MinArgs {
|
|
|
|
_ = cmd.Usage()
|
|
|
|
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name(), MinArgs)
|
|
|
|
os.Exit(1)
|
|
|
|
} else if len(args) > MaxArgs {
|
|
|
|
_ = cmd.Usage()
|
|
|
|
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name(), MaxArgs)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-01 09:49:47 +01:00
|
|
|
// StartStats prints the stats every statsInterval
|
2016-08-04 23:18:56 +02:00
|
|
|
//
|
|
|
|
// It returns a channel which should be closed to stop the stats.
|
2016-12-01 09:49:47 +01:00
|
|
|
func StartStats() chan struct{} {
|
2016-08-04 23:18:56 +02:00
|
|
|
stopStats := make(chan struct{})
|
|
|
|
if *statsInterval > 0 {
|
|
|
|
go func() {
|
|
|
|
ticker := time.NewTicker(*statsInterval)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
fs.Stats.Log()
|
|
|
|
case <-stopStats:
|
|
|
|
ticker.Stop()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
return stopStats
|
|
|
|
}
|
|
|
|
|
|
|
|
// initConfig is run by cobra after initialising the flags
|
|
|
|
func initConfig() {
|
2017-02-10 14:28:06 +01:00
|
|
|
// Start the logger
|
|
|
|
fs.InitLogging()
|
2016-08-04 23:18:56 +02:00
|
|
|
|
|
|
|
// Load the rest of the config now we have started the logger
|
|
|
|
fs.LoadConfig()
|
|
|
|
|
|
|
|
// Write the args for debug purposes
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
|
2016-08-04 23:18:56 +02:00
|
|
|
|
|
|
|
// Setup CPU profiling if desired
|
|
|
|
if *cpuProfile != "" {
|
2017-02-09 18:08:51 +01:00
|
|
|
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
|
2016-08-04 23:18:56 +02:00
|
|
|
f, err := os.Create(*cpuProfile)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
err = pprof.StartCPUProfile(f)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
2017-02-20 17:33:45 +01:00
|
|
|
AtExit(func() {
|
|
|
|
pprof.StopCPUProfile()
|
|
|
|
})
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Setup memory profiling if desired
|
|
|
|
if *memProfile != "" {
|
2017-02-20 17:33:45 +01:00
|
|
|
AtExit(func() {
|
2017-02-09 18:08:51 +01:00
|
|
|
fs.Infof(nil, "Saving Memory profile %q\n", *memProfile)
|
2016-08-04 23:18:56 +02:00
|
|
|
f, err := os.Create(*memProfile)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
err = pprof.WriteHeapProfile(f)
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
err = f.Close()
|
|
|
|
if err != nil {
|
|
|
|
fs.Stats.Error()
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
2017-02-20 17:33:45 +01:00
|
|
|
})
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|
2016-11-22 05:04:05 +01:00
|
|
|
|
|
|
|
if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false {
|
2017-02-09 12:01:20 +01:00
|
|
|
fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")
|
2016-11-22 05:04:05 +01:00
|
|
|
fs.Config.DataRateUnit = "bytes"
|
|
|
|
} else {
|
|
|
|
fs.Config.DataRateUnit = *dataRateUnit
|
|
|
|
}
|
2016-08-04 23:18:56 +02:00
|
|
|
}
|