Remove filesystem flags and put in config file with editor

.gitignore

@@ -3,3 +3,4 @@
 test-env*
 _junk/
 rclone
+upload

drive/fs.go

@@ -22,30 +22,52 @@ package drive
 // * files with / in name
 
 import (
-	"code.google.com/p/goauth2/oauth"
-	"code.google.com/p/google-api-go-client/drive/v2"
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/ncw/rclone/fs"
 	"io"
 	"log"
 	"mime"
 	"net/http"
 	"os"
 	"path"
-	"regexp"
 	"strings"
 	"sync"
 	"time"
-)
 
-// Pattern to match a drive url
+	"code.google.com/p/goauth2/oauth"
-var Match = regexp.MustCompile(`^drive://(.*)$`)
+	"code.google.com/p/google-api-go-client/drive/v2"
+	"github.com/ncw/rclone/fs"
+)
 
 // Register with Fs
 func init() {
-	fs.Register(Match, NewFs)
+	fs.Register(&fs.FsInfo{
+		Name:  "drive",
+		NewFs: NewFs,
+		Options: []fs.Option{{
+			Name: "client_id",
+			Help: "Google Application Client Id.",
+			Examples: []fs.OptionExample{{
+				Value: "202264815644.apps.googleusercontent.com",
+				Help:  "rclone's client id - use this or your own if you want",
+			}},
+		}, {
+			Name: "client_secret",
+			Help: "Google Application Client Secret.",
+			Examples: []fs.OptionExample{{
+				Value: "X4Z3ca8xfWDb1Voo-F9a7ZxJ",
+				Help:  "rclone's client secret - use this or your own if you want",
+			}},
+		}, {
+			Name: "token_file",
+			Help: "Path to store token file.",
+			Examples: []fs.OptionExample{{
+				Value: path.Join(fs.HomeDir, ".gdrive-token-file"),
+				Help:  "Suggested path for token file",
+			}},
+		}},
+	})
 }
 
 // FsDrive represents a remote drive server
@@ -128,11 +150,8 @@ const (
 // Globals
 var (
 	// Flags
-	driveClientId     = flag.String("drive-client-id", os.Getenv("GDRIVE_CLIENT_ID"), "Auth URL for server. Defaults to environment var GDRIVE_CLIENT_ID.")
+	driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.")
-	driveClientSecret = flag.String("drive-client-secret", os.Getenv("GDRIVE_CLIENT_SECRET"), "User name. Defaults to environment var GDRIVE_CLIENT_SECRET.")
+	driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.")
-	driveTokenFile    = flag.String("drive-token-file", os.Getenv("GDRIVE_TOKEN_FILE"), "API key (password). Defaults to environment var GDRIVE_TOKEN_FILE.")
-	driveAuthCode     = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.")
-	driveFullList     = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.")
 )
 
 // String converts this FsDrive to a string
@@ -142,13 +161,7 @@ func (f *FsDrive) String() string {
 
 // parseParse parses a drive 'url'
 func parseDrivePath(path string) (root string, err error) {
-	parts := Match.FindAllStringSubmatch(path, -1)
+	root = strings.Trim(root, "/")
-	if len(parts) != 1 || len(parts[0]) != 2 {
-		err = fmt.Errorf("Couldn't parse drive url %q", path)
-	} else {
-		root = parts[0][1]
-		root = strings.Trim(root, "/")
-	}
 	return
 }
 
@@ -222,26 +235,29 @@ func MakeNewToken(t *oauth.Transport) error {
 }
 
 // NewFs contstructs an FsDrive from the path, container:path
-func NewFs(path string) (fs.Fs, error) {
+func NewFs(name, path string) (fs.Fs, error) {
-	if *driveClientId == "" {
+	clientId := fs.ConfigFile.MustValue(name, "client_id")
-		return nil, errors.New("Need -drive-client-id or environmental variable GDRIVE_CLIENT_ID")
+	if clientId == "" {
+		return nil, errors.New("client_id not found")
 	}
-	if *driveClientSecret == "" {
+	clientSecret := fs.ConfigFile.MustValue(name, "client_secret")
-		return nil, errors.New("Need -drive-client-secret or environmental variable GDRIVE_CLIENT_SECRET")
+	if clientSecret == "" {
+		return nil, errors.New("client_secret not found")
 	}
-	if *driveTokenFile == "" {
+	tokenFile := fs.ConfigFile.MustValue(name, "token_file")
-		return nil, errors.New("Need -drive-token-file or environmental variable GDRIVE_TOKEN_FILE")
+	if tokenFile == "" {
+		return nil, errors.New("token-file not found")
 	}
 
 	// Settings for authorization.
 	var driveConfig = &oauth.Config{
-		ClientId:     *driveClientId,
+		ClientId:     clientId,
-		ClientSecret: *driveClientSecret,
+		ClientSecret: clientSecret,
 		Scope:        "https://www.googleapis.com/auth/drive",
 		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
 		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
 		TokenURL:     "https://accounts.google.com/o/oauth2/token",
-		TokenCache:   oauth.CacheFile(*driveTokenFile),
+		TokenCache:   oauth.CacheFile(tokenFile),
 	}
 
 	root, err := parseDrivePath(path)

fs/config.go

@@ -0,0 +1,260 @@
+// Read and write the config file
+package fs
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/user"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Unknwon/goconfig"
+)
+
+const (
+	configFileName = ".rclone.conf"
+)
+
+// Global
+var (
+	// Config file
+	ConfigFile *goconfig.ConfigFile
+	// Config file path
+	ConfigPath string
+	// Global config
+	Config = &ConfigInfo{}
+	// Home directory
+	HomeDir string
+	// Flags
+	verbose      = flag.Bool("verbose", false, "Print lots more stuff")
+	quiet        = flag.Bool("quiet", false, "Print as little stuff as possible")
+	modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
+	checkers     = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
+	transfers    = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
+)
+
+// Filesystem config options
+type ConfigInfo struct {
+	Verbose      bool
+	Quiet        bool
+	ModifyWindow time.Duration
+	Checkers     int
+	Transfers    int
+}
+
+// Loads the config file
+func LoadConfig() {
+	// Read some flags if set
+	//
+	// FIXME read these from the config file too
+	Config.Verbose = *verbose
+	Config.Quiet = *quiet
+	Config.ModifyWindow = *modifyWindow
+	Config.Checkers = *checkers
+	Config.Transfers = *transfers
+
+	// Find users home directory
+	usr, err := user.Current()
+	if err != nil {
+		log.Printf("Couldn't find home directory: %v", err)
+		return
+	}
+	HomeDir = usr.HomeDir
+	ConfigPath = path.Join(HomeDir, configFileName)
+
+	// Load configuration file.
+	ConfigFile, err = goconfig.LoadConfigFile(ConfigPath)
+	if err != nil {
+		log.Printf("Failed to load config file %v - using defaults", ConfigPath)
+	}
+}
+
+// Save configuration file.
+func SaveConfig() {
+	err := goconfig.SaveConfigFile(ConfigFile, ConfigPath)
+	if err != nil {
+		log.Fatalf("Failed to save config file: %v", err)
+	}
+}
+
+// Show an overview of the config file
+func ShowConfig() {
+	remotes := ConfigFile.GetSectionList()
+	sort.Strings(remotes)
+	fmt.Printf("%-20s %s\n", "Name", "Type")
+	fmt.Printf("%-20s %s\n", "====", "====")
+	for _, remote := range remotes {
+		fmt.Printf("%-20s %s\n", remote, ConfigFile.MustValue(remote, "type"))
+	}
+}
+
+// ChooseRemote chooses a remote name
+func ChooseRemote() string {
+	remotes := ConfigFile.GetSectionList()
+	sort.Strings(remotes)
+	return Choose("remote", remotes, nil, false)
+}
+
+// Read some input
+func ReadLine() string {
+	buf := bufio.NewReader(os.Stdin)
+	line, err := buf.ReadString('\n')
+	if err != nil {
+		log.Fatalf("Failed to read line: %v", err)
+	}
+	return strings.TrimSpace(line)
+}
+
+// Command - choose one
+func Command(commands []string) int {
+	opts := []string{}
+	for _, text := range commands {
+		fmt.Printf("%c) %s\n", text[0], text[1:])
+		opts = append(opts, text[:1])
+	}
+	optString := strings.Join(opts, "")
+	optHelp := strings.Join(opts, "/")
+	for {
+		fmt.Printf("%s> ", optHelp)
+		result := strings.ToLower(ReadLine())
+		if len(result) != 1 {
+			continue
+		}
+		i := strings.IndexByte(optString, result[0])
+		if i >= 0 {
+			return i
+		}
+	}
+}
+
+// Choose one of the defaults or type a new string if newOk is set
+func Choose(what string, defaults, help []string, newOk bool) string {
+	fmt.Printf("Choose a number from below")
+	if newOk {
+		fmt.Printf(", or type in your own value")
+	}
+	fmt.Println()
+	for i, text := range defaults {
+		if help != nil {
+			parts := strings.Split(help[i], "\n")
+			for _, part := range parts {
+				fmt.Printf(" * %s\n", part)
+			}
+		}
+		fmt.Printf("%2d) %s\n", i+1, text)
+	}
+	for {
+		fmt.Printf("%s> ", what)
+		result := ReadLine()
+		i, err := strconv.Atoi(result)
+		if err != nil {
+			if newOk {
+				return result
+			}
+			continue
+		}
+		if i >= 1 && i <= len(defaults) {
+			return defaults[i-1]
+		}
+	}
+}
+
+// Show the contents of the remote
+func ShowRemote(name string) {
+	fmt.Printf("--------------------\n")
+	fmt.Printf("[%s]\n", name)
+	for _, key := range ConfigFile.GetKeyList(name) {
+		fmt.Printf("%s = %s\n", key, ConfigFile.MustValue(name, key))
+	}
+	fmt.Printf("--------------------\n")
+}
+
+// Print the contents of the remote and ask if it is OK
+func OkRemote(name string) bool {
+	ShowRemote(name)
+	switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i {
+	case 0:
+		return true
+	case 1:
+		return false
+	case 2:
+		ConfigFile.DeleteSection(name)
+		return true
+	default:
+		log.Printf("Bad choice %d", i)
+	}
+	return false
+}
+
+// Make a new remote
+func NewRemote(name string) {
+	fmt.Printf("What type of source is it?\n")
+	types := []string{}
+	for _, item := range fsRegistry {
+		types = append(types, item.Name)
+	}
+	newType := Choose("type", types, nil, false)
+	ConfigFile.SetValue(name, "type", newType)
+	fs, err := Find(newType)
+	if err != nil {
+		log.Fatalf("Failed to find fs: %v", err)
+	}
+	for _, option := range fs.Options {
+		ConfigFile.SetValue(name, option.Name, option.Choose())
+	}
+	if OkRemote(name) {
+		SaveConfig()
+		return
+	}
+	EditRemote(name)
+}
+
+// Edit a remote
+func EditRemote(name string) {
+	ShowRemote(name)
+	fmt.Printf("Edit remote\n")
+	for {
+		for _, key := range ConfigFile.GetKeyList(name) {
+			value := ConfigFile.MustValue(name, key)
+			fmt.Printf("Press enter to accept current value, or type in a new one\n")
+			fmt.Printf("%s = %s>", key, value)
+			newValue := ReadLine()
+			if newValue != "" {
+				ConfigFile.SetValue(name, key, newValue)
+			}
+		}
+		if OkRemote(name) {
+			break
+		}
+	}
+	SaveConfig()
+}
+
+// Edit the config file interactively
+func EditConfig() {
+	for {
+		fmt.Printf("Current remotes:\n\n")
+		ShowConfig()
+		fmt.Printf("\n")
+		switch i := Command([]string{"eEdit existing remote", "nNew remote", "dDelete remote", "qQuit config"}); i {
+		case 0:
+			name := ChooseRemote()
+			EditRemote(name)
+		case 1:
+			fmt.Printf("name> ")
+			name := ReadLine()
+			NewRemote(name)
+		case 2:
+			name := ChooseRemote()
+			ConfigFile.DeleteSection(name)
+		case 3:
+			return
+		}
+	}
+}

fs/fs.go

@@ -12,41 +12,52 @@ import (
 
 // Globals
 var (
-	// Global config
-	Config = &ConfigInfo{}
 	// Filesystem registry
-	fsRegistry []registryItem
+	fsRegistry []*FsInfo
 )
 
-// Filesystem config options
+// Filesystem info
-type ConfigInfo struct {
+type FsInfo struct {
-	Verbose      bool
+	Name    string                           // name of this fs
-	Quiet        bool
+	NewFs   func(string, string) (Fs, error) // create a new file system
-	ModifyWindow time.Duration
+	Options []Option
-	Checkers     int
-	Transfers    int
 }
 
-// Filesystem registry item
+// An options for a Fs
-type registryItem struct {
+type Option struct {
-	match *regexp.Regexp           // if this matches then can call newFs
+	Name     string
-	newFs func(string) (Fs, error) // create a new file system
+	Help     string
+	Optional bool
+	Examples []OptionExample
+}
+
+// An example for an option
+type OptionExample struct {
+	Value string
+	Help  string
+}
+
+// Choose an option
+func (o *Option) Choose() string {
+	fmt.Println(o.Help)
+	if len(o.Examples) > 0 {
+		var values []string
+		var help []string
+		for _, example := range o.Examples {
+			values = append(values, example.Value)
+			help = append(help, example.Help)
+		}
+		return Choose(o.Name, values, help, true)
+	}
+	fmt.Printf("%s> ", o.Name)
+	return ReadLine()
 }
 
 // Register a filesystem
 //
-// If a path matches with match then can call newFs on it
-//
-// Pass with match nil goes last and matches everything (used by local fs)
-//
 // Fs modules  should use this in an init() function
-func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
+func Register(info *FsInfo) {
-	fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
+	fsRegistry = append(fsRegistry, info)
-	// Keep one nil match at the end
-	last := len(fsRegistry) - 1
-	if last >= 1 && fsRegistry[last-1].match == nil {
-		fsRegistry[last], fsRegistry[last-1] = fsRegistry[last-1], fsRegistry[last]
-	}
 }
 
 // A Filesystem, describes the local filesystem and the remote object store
@@ -136,16 +147,42 @@ type Dir struct {
 // A channel of Dir objects
 type DirChan chan *Dir
 
-// NewFs makes a new Fs object from the path
+// Pattern to match a url
+var matcher = regexp.MustCompile(`^([\w_-]+)://(.*)$`)
+
+// Finds a FsInfo object for the name passed in
 //
-// FIXME make more generic
+// Services are looked up in the config file
-func NewFs(path string) (Fs, error) {
+func Find(name string) (*FsInfo, error) {
 	for _, item := range fsRegistry {
-		if item.match == nil || item.match.MatchString(path) {
+		if item.Name == name {
-			return item.newFs(path)
+			return item, nil
 		}
 	}
-	panic("Not found") // FIXME
+	return nil, fmt.Errorf("Didn't find filing system for %q", name)
+}
+
+// NewFs makes a new Fs object from the path
+//
+// The path is of the form service://path
+//
+// Services are looked up in the config file
+func NewFs(path string) (Fs, error) {
+	parts := matcher.FindStringSubmatch(path)
+	fsName, configName, fsPath := "local", "local", path
+	if parts != nil {
+		configName, fsPath = parts[1], parts[2]
+		var err error
+		fsName, err = ConfigFile.GetValue(configName, "type")
+		if err != nil {
+			return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
+		}
+	}
+	fs, err := Find(fsName)
+	if err != nil {
+		return nil, err
+	}
+	return fs.NewFs(configName, fsPath)
 }
 
 // Write debuging output for this Object

local/fs.go

@@ -4,7 +4,6 @@ package local
 import (
 	"crypto/md5"
 	"fmt"
-	"github.com/ncw/rclone/fs"
 	"io"
 	"io/ioutil"
 	"log"
@@ -13,11 +12,16 @@ import (
 	"path/filepath"
 	"sync"
 	"time"
+
+	"github.com/ncw/rclone/fs"
 )
 
 // Register with Fs
 func init() {
-	fs.Register(nil, NewFs)
+	fs.Register(&fs.FsInfo{
+		Name:  "local",
+		NewFs: NewFs,
+	})
 }
 
 // FsLocal represents a local filesystem rooted at root
@@ -37,7 +41,7 @@ type FsObjectLocal struct {
 // ------------------------------------------------------------
 
 // NewFs contstructs an FsLocal from the path
-func NewFs(root string) (fs.Fs, error) {
+func NewFs(name, root string) (fs.Fs, error) {
 	root = path.Clean(root)
 	f := &FsLocal{root: root}
 	return f, nil

rclone.go

@@ -6,7 +6,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/ncw/rclone/fs"
 	"log"
 	"os"
 	"runtime"
@@ -14,6 +13,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/ncw/rclone/fs"
 	// Active file systems
 	_ "github.com/ncw/rclone/drive"
 	_ "github.com/ncw/rclone/local"
@@ -25,13 +26,8 @@ import (
 var (
 	// Flags
 	cpuprofile    = flag.String("cpuprofile", "", "Write cpu profile to file")
-	verbose       = flag.Bool("verbose", false, "Print lots more stuff")
-	quiet         = flag.Bool("quiet", false, "Print as little stuff as possible")
 	dry_run       = flag.Bool("dry-run", false, "Do a trial run with no permanent changes")
-	checkers      = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
-	transfers     = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
 	statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
-	modifyWindow  = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
 )
 
 // A pair of fs.Objects
@@ -105,17 +101,17 @@ func CopyFs(fdst, fsrc fs.Fs) {
 	}
 
 	to_be_checked := fsrc.List()
-	to_be_uploaded := make(fs.ObjectsChan, *transfers)
+	to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
 
 	var checkerWg sync.WaitGroup
-	checkerWg.Add(*checkers)
+	checkerWg.Add(fs.Config.Checkers)
-	for i := 0; i < *checkers; i++ {
+	for i := 0; i < fs.Config.Checkers; i++ {
 		go Checker(to_be_checked, to_be_uploaded, fdst, &checkerWg)
 	}
 
 	var copierWg sync.WaitGroup
-	copierWg.Add(*transfers)
+	copierWg.Add(fs.Config.Transfers)
-	for i := 0; i < *transfers; i++ {
+	for i := 0; i < fs.Config.Transfers; i++ {
 		go Copier(to_be_uploaded, fdst, &copierWg)
 	}
 
@@ -129,8 +125,8 @@ func CopyFs(fdst, fsrc fs.Fs) {
 // Delete all the files passed in the channel
 func DeleteFiles(to_be_deleted fs.ObjectsChan) {
 	var wg sync.WaitGroup
-	wg.Add(*transfers)
+	wg.Add(fs.Config.Transfers)
-	for i := 0; i < *transfers; i++ {
+	for i := 0; i < fs.Config.Transfers; i++ {
 		go func() {
 			defer wg.Done()
 			for dst := range to_be_deleted {
@@ -173,18 +169,18 @@ func Sync(fdst, fsrc fs.Fs) {
 	}
 
 	// Read source files checking them off against dest files
-	to_be_checked := make(PairFsObjectsChan, *transfers)
+	to_be_checked := make(PairFsObjectsChan, fs.Config.Transfers)
-	to_be_uploaded := make(fs.ObjectsChan, *transfers)
+	to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
 
 	var checkerWg sync.WaitGroup
-	checkerWg.Add(*checkers)
+	checkerWg.Add(fs.Config.Checkers)
-	for i := 0; i < *checkers; i++ {
+	for i := 0; i < fs.Config.Checkers; i++ {
 		go PairChecker(to_be_checked, to_be_uploaded, &checkerWg)
 	}
 
 	var copierWg sync.WaitGroup
-	copierWg.Add(*transfers)
+	copierWg.Add(fs.Config.Transfers)
-	for i := 0; i < *transfers; i++ {
+	for i := 0; i < fs.Config.Transfers; i++ {
 		go Copier(to_be_uploaded, fdst, &copierWg)
 	}
 
@@ -215,7 +211,7 @@ func Sync(fdst, fsrc fs.Fs) {
 	}
 
 	// Delete the spare files
-	toDelete := make(fs.ObjectsChan, *transfers)
+	toDelete := make(fs.ObjectsChan, fs.Config.Transfers)
 	go func() {
 		for _, fs := range delFiles {
 			toDelete <- fs
@@ -262,7 +258,7 @@ func Check(fdst, fsrc fs.Fs) {
 		log.Printf(remote)
 	}
 
-	checks := make(chan []fs.Object, *transfers)
+	checks := make(chan []fs.Object, fs.Config.Transfers)
 	go func() {
 		for _, check := range commonFiles {
 			checks <- check
@@ -271,8 +267,8 @@ func Check(fdst, fsrc fs.Fs) {
 	}()
 
 	var checkerWg sync.WaitGroup
-	checkerWg.Add(*checkers)
+	checkerWg.Add(fs.Config.Checkers)
-	for i := 0; i < *checkers; i++ {
+	for i := 0; i < fs.Config.Checkers; i++ {
 		go func() {
 			defer checkerWg.Done()
 			for check := range checks {
@@ -309,8 +305,8 @@ func Check(fdst, fsrc fs.Fs) {
 func List(f, _ fs.Fs) {
 	in := f.List()
 	var wg sync.WaitGroup
-	wg.Add(*checkers)
+	wg.Add(fs.Config.Checkers)
-	for i := 0; i < *checkers; i++ {
+	for i := 0; i < fs.Config.Checkers; i++ {
 		go func() {
 			defer wg.Done()
 			for o := range in {
@@ -370,117 +366,128 @@ func purge(fdst, fsrc fs.Fs) {
 	}
 }
 
+// Edits the config file
+func EditConfig(fdst, fsrc fs.Fs) {
+	fs.EditConfig()
+}
+
 type Command struct {
-	name             string
+	Name     string
-	help             string
+	Help     string
-	run              func(fdst, fsrc fs.Fs)
+	ArgsHelp string
-	minArgs, maxArgs int
+	Run      func(fdst, fsrc fs.Fs)
+	MinArgs  int
+	MaxArgs  int
+	NoStats  bool
 }
 
 // checkArgs checks there are enough arguments and prints a message if not
 func (cmd *Command) checkArgs(args []string) {
-	if len(args) < cmd.minArgs {
+	if len(args) < cmd.MinArgs {
 		syntaxError()
-		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.name, cmd.minArgs)
+		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
 		os.Exit(1)
-	} else if len(args) > cmd.maxArgs {
+	} else if len(args) > cmd.MaxArgs {
 		syntaxError()
-		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.name, cmd.maxArgs)
+		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
 		os.Exit(1)
 	}
 }
 
 var Commands = []Command{
 	{
-		"copy",
+		Name:     "copy",
-		`<source> <destination>
+		ArgsHelp: "source://path dest://path",
-
+		Help: `
         Copy the source to the destination.  Doesn't transfer
         unchanged files, testing first by modification time then by
-        MD5SUM.  Doesn't delete files from the destination.
+        MD5SUM.  Doesn't delete files from the destination.`,
-
+		Run:     CopyFs,
-`,
+		MinArgs: 2,
-		CopyFs,
+		MaxArgs: 2,
-		2, 2,
 	},
 	{
-		"sync",
+		Name:     "sync",
-		`<source> <destination>
+		ArgsHelp: "source://path dest://path",
-
+		Help: `
         Sync the source to the destination.  Doesn't transfer
         unchanged files, testing first by modification time then by
         MD5SUM.  Deletes any files that exist in source that don't
         exist in destination. Since this can cause data loss, test
         first with the -dry-run flag.`,
-
+		Run:     Sync,
-		Sync,
+		MinArgs: 2,
-		2, 2,
+		MaxArgs: 2,
 	},
 	{
-		"ls",
+		Name:     "ls",
-		`[<path>]
+		ArgsHelp: "[remote://path]",
-
+		Help: `
         List all the objects in the the path.`,
-
+		Run:     List,
-		List,
+		MinArgs: 1,
-		1, 1,
+		MaxArgs: 1,
 	},
 	{
-		"lsd",
+		Name:     "lsd",
-		`[<path>]
+		ArgsHelp: "[remote://path]",
-
+		Help: `
         List all directoryes/objects/buckets in the the path.`,
-
+		Run:     ListDir,
-		ListDir,
+		MinArgs: 1,
-		1, 1,
+		MaxArgs: 1,
 	},
 	{
-		"mkdir",
+		Name:     "mkdir",
-		`<path>
+		ArgsHelp: "remote://path",
-
+		Help: `
         Make the path if it doesn't already exist`,
-
+		Run:     mkdir,
-		mkdir,
+		MinArgs: 1,
-		1, 1,
+		MaxArgs: 1,
 	},
 	{
-		"rmdir",
+		Name:     "rmdir",
-		`<path>
+		ArgsHelp: "remote://path",
-
+		Help: `
         Remove the path.  Note that you can't remove a path with
         objects in it, use purge for that.`,
-
+		Run:     rmdir,
-		rmdir,
+		MinArgs: 1,
-		1, 1,
+		MaxArgs: 1,
 	},
 	{
-		"purge",
+		Name:     "purge",
-		`<path>
+		ArgsHelp: "remote://path",
-
+		Help: `
         Remove the path and all of its contents.`,
-
+		Run:     purge,
-		purge,
+		MinArgs: 1,
-		1, 1,
+		MaxArgs: 1,
 	},
 	{
-		"check",
+		Name:     "check",
-		`<source> <destination>
+		ArgsHelp: "source://path dest://path",
-
+		Help: `
         Checks the files in the source and destination match.  It
         compares sizes and MD5SUMs and prints a report of files which
         don't match.  It doesn't alter the source or destination.`,
-
+		Run:     Check,
-		Check,
+		MinArgs: 2,
-		2, 2,
+		MaxArgs: 2,
 	},
 	{
-		"help",
+		Name: "config",
-		`
+		Help: `
-
+        Enter an interactive configuration session.`,
+		Run:     EditConfig,
+		NoStats: true,
+	},
+	{
+		Name: "help",
+		Help: `
         This help.`,
-
+		NoStats: true,
-		nil,
-		0, 0,
 	},
 }
 
@@ -495,7 +502,8 @@ Subcommands:
 `)
 	for i := range Commands {
 		cmd := &Commands[i]
-		fmt.Fprintf(os.Stderr, "    %s: %s\n\n", cmd.name, cmd.help)
+		fmt.Fprintf(os.Stderr, "    %s %s\n", cmd.Name, cmd.ArgsHelp)
+		fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
 	}
 
 	fmt.Fprintf(os.Stderr, "Options:\n")
@@ -517,13 +525,7 @@ func main() {
 	flag.Parse()
 	args := flag.Args()
 	runtime.GOMAXPROCS(runtime.NumCPU())
-
+	fs.LoadConfig()
-	// Pass on some flags to fs.Config
-	fs.Config.Verbose = *verbose
-	fs.Config.Quiet = *quiet
-	fs.Config.ModifyWindow = *modifyWindow
-	fs.Config.Checkers = *checkers
-	fs.Config.Transfers = *transfers
 
 	// Setup profiling if desired
 	if *cpuprofile != "" {
@@ -548,10 +550,10 @@ func main() {
 	for i := range Commands {
 		command := &Commands[i]
 		// exact command name found - use that
-		if command.name == cmd {
+		if command.Name == cmd {
 			found = command
 			break
-		} else if strings.HasPrefix(command.name, cmd) {
+		} else if strings.HasPrefix(command.Name, cmd) {
 			if found != nil {
 				fs.Stats.Error()
 				log.Fatalf("Not unique - matches multiple commands %q", cmd)
@@ -572,14 +574,14 @@ func main() {
 		fdst, err = fs.NewFs(args[0])
 		if err != nil {
 			fs.Stats.Error()
-			log.Fatal("Failed to create file system: ", err)
+			log.Fatalf("Failed to create file system for %q: %v", args[0], err)
 		}
 	}
 	if len(args) >= 2 {
 		fsrc, err = fs.NewFs(args[1])
 		if err != nil {
 			fs.Stats.Error()
-			log.Fatal("Failed to create destination file system: ", err)
+			log.Fatalf("Failed to create destination file system for %q: %v", args[1], err)
 		}
 		fsrc, fdst = fdst, fsrc
 	}
@@ -599,22 +601,30 @@ func main() {
 			fs.Config.ModifyWindow = precision
 		}
 	}
-	log.Printf("Modify window is %s\n", fs.Config.ModifyWindow)
+	if fs.Config.Verbose {
+		log.Printf("Modify window is %s\n", fs.Config.ModifyWindow)
+	}
 
 	// Print the stats every statsInterval
-	go func() {
+	if !found.NoStats {
-		ch := time.Tick(*statsInterval)
+		go func() {
-		for {
+			ch := time.Tick(*statsInterval)
-			<-ch
+			for {
-			fs.Stats.Log()
+				<-ch
-		}
+				fs.Stats.Log()
-	}()
+			}
+		}()
+	}
 
 	// Run the actual command
-	if found.run != nil {
+	if found.Run != nil {
-		found.run(fdst, fsrc)
+		found.Run(fdst, fsrc)
-		fmt.Println(fs.Stats)
+		if !found.NoStats {
-		log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
+			fmt.Println(fs.Stats)
+		}
+		if fs.Config.Verbose {
+			log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
+		}
 		if fs.Stats.Errored() {
 			os.Exit(1)
 		}

s3/fs.go

@@ -5,30 +5,99 @@ package s3
 
 import (
 	"errors"
-	"flag"
 	"fmt"
-	"github.com/ncw/goamz/aws"
-	"github.com/ncw/goamz/s3"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/swift"
 	"io"
 	"log"
 	"mime"
 	"net/http"
-	"os"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
-)
 
-// Pattern to match a s3 url
+	"github.com/ncw/goamz/aws"
-var Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`)
+	"github.com/ncw/goamz/s3"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/swift"
+)
 
 // Register with Fs
 func init() {
-	fs.Register(Match, NewFs)
+	fs.Register(&fs.FsInfo{
+		Name:  "s3",
+		NewFs: NewFs,
+		// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
+		Options: []fs.Option{{
+			Name: "access_key_id",
+			Help: "AWS Access Key ID.",
+		}, {
+			Name: "secret_access_key",
+			Help: "AWS Secret Access Key (password). ",
+		}, {
+			Name: "endpoint",
+			Help: "Endpoint for S3 API.",
+			Examples: []fs.OptionExample{{
+				Value: "https://s3.amazonaws.com/",
+				Help:  "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.",
+			}, {
+				Value: "https://s3-external-1.amazonaws.com",
+				Help:  "US Region, Northern Virginia only.\nLeave location constraint empty.",
+			}, {
+				Value: "https://s3-us-west-2.amazonaws.com",
+				Help:  "US West (Oregon) Region\nNeeds location constraint us-west-2.",
+			}, {
+				Value: "https://s3-us-west-1.amazonaws.com",
+				Help:  "US West (Northern California) Region\nNeeds location constraint us-west-1.",
+			}, {
+				Value: "https://s3-eu-west-1.amazonaws.com",
+				Help:  "EU (Ireland) Region Region\nNeeds location constraint EU or eu-west-1.",
+			}, {
+				Value: "https://s3-ap-southeast-1.amazonaws.com",
+				Help:  "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.",
+			}, {
+				Value: "https://s3-ap-southeast-2.amazonaws.com",
+				Help:  "Asia Pacific (Sydney) Region\nNeeds location constraint .",
+			}, {
+				Value: "https://s3-ap-northeast-1.amazonaws.com",
+				Help:  "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.",
+			}, {
+				Value: "https://s3-sa-east-1.amazonaws.com",
+				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
+			}},
+		}, {
+			Name: "location_constraint",
+			Help: "Location constraint - must be set to match the Endpoint.",
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "Empty for US Region, Northern Virginia or Pacific Northwest.",
+			}, {
+				Value: "us-west-2",
+				Help:  "US West (Oregon) Region.",
+			}, {
+				Value: "us-west-1",
+				Help:  "US West (Northern California) Region.",
+			}, {
+				Value: "eu-west-1",
+				Help:  "EU (Ireland) Region.",
+			}, {
+				Value: "EU",
+				Help:  "EU Region.",
+			}, {
+				Value: "ap-southeast-1",
+				Help:  "Asia Pacific (Singapore) Region.",
+			}, {
+				Value: "ap-southeast-2",
+				Help:  "Asia Pacific (Sydney) Region.",
+			}, {
+				Value: "ap-northeast-1",
+				Help:  "Asia Pacific (Tokyo) Region.",
+			}, {
+				Value: "sa-east-1",
+				Help:  "South America (Sao Paulo) Region.",
+			}},
+		}},
+	})
 }
 
 // Constants
@@ -60,57 +129,54 @@ type FsObjectS3 struct {
 
 // ------------------------------------------------------------
 
-// Globals
-var (
-	// Flags
-	awsAccessKeyId     = flag.String("aws-access-key-id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID.")
-	awsSecretAccessKey = flag.String("aws-secret-access-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY.")
-	// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
-	s3Endpoint           = flag.String("s3-endpoint", os.Getenv("S3_ENDPOINT"), "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/.")
-	s3LocationConstraint = flag.String("s3-location-constraint", os.Getenv("S3_LOCATION_CONSTRAINT"), "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT.")
-)
-
 // String converts this FsS3 to a string
 func (f *FsS3) String() string {
 	return fmt.Sprintf("S3 bucket %s", f.bucket)
 }
 
+// Pattern to match a s3 path
+var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
+
 // parseParse parses a s3 'url'
 func s3ParsePath(path string) (bucket, directory string, err error) {
-	parts := Match.FindAllStringSubmatch(path, -1)
+	parts := matcher.FindStringSubmatch(path)
-	if len(parts) != 1 || len(parts[0]) != 3 {
+	if parts == nil {
-		err = fmt.Errorf("Couldn't parse s3 url %q", path)
+		err = fmt.Errorf("Couldn't parse bucket out of s3 path %q", path)
 	} else {
-		bucket, directory = parts[0][1], parts[0][2]
+		bucket, directory = parts[1], parts[2]
 		directory = strings.Trim(directory, "/")
 	}
 	return
 }
 
 // s3Connection makes a connection to s3
-func s3Connection() (*s3.S3, error) {
+func s3Connection(name string) (*s3.S3, error) {
 	// Make the auth
-	if *awsAccessKeyId == "" {
+	accessKeyId := fs.ConfigFile.MustValue(name, "access_key_id")
-		return nil, errors.New("Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID")
+	if accessKeyId == "" {
+		return nil, errors.New("access_key_id not found")
 	}
-	if *awsSecretAccessKey == "" {
+	secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
-		return nil, errors.New("Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY")
+	if secretAccessKey == "" {
+		return nil, errors.New("secret_access_key not found")
 	}
-	auth := aws.Auth{AccessKey: *awsAccessKeyId, SecretKey: *awsSecretAccessKey}
+	auth := aws.Auth{AccessKey: accessKeyId, SecretKey: secretAccessKey}
 
 	// FIXME look through all the regions by name and use one of them if found
 
 	// Synthesize the region
-	if *s3Endpoint == "" {
+	s3Endpoint := fs.ConfigFile.MustValue(name, "endpoint")
-		*s3Endpoint = "https://s3.amazonaws.com/"
+	if s3Endpoint == "" {
+		s3Endpoint = "https://s3.amazonaws.com/"
 	}
 	region := aws.Region{
 		Name:                 "s3",
-		S3Endpoint:           *s3Endpoint,
+		S3Endpoint:           s3Endpoint,
 		S3LocationConstraint: false,
 	}
-	if *s3LocationConstraint != "" {
+	s3LocationConstraint := fs.ConfigFile.MustValue(name, "location_constraint")
-		region.Name = *s3LocationConstraint
+	if s3LocationConstraint != "" {
+		region.Name = s3LocationConstraint
 		region.S3LocationConstraint = true
 	}
 
@@ -119,7 +185,7 @@ func s3Connection() (*s3.S3, error) {
 }
 
 // NewFsS3 contstructs an FsS3 from the path, bucket:path
-func NewFs(path string) (fs.Fs, error) {
+func NewFs(name, path string) (fs.Fs, error) {
 	bucket, directory, err := s3ParsePath(path)
 	if err != nil {
 		return nil, err
@@ -127,7 +193,7 @@ func NewFs(path string) (fs.Fs, error) {
 	if directory != "" {
 		return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory)
 	}
-	c, err := s3Connection()
+	c, err := s3Connection(name)
 	if err != nil {
 		return nil, err
 	}

swift/fs.go

@@ -5,24 +5,51 @@ package swift
 
 import (
 	"errors"
-	"flag"
 	"fmt"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/swift"
 	"io"
 	"log"
-	"os"
 	"regexp"
 	"strings"
 	"time"
-)
 
-// Pattern to match a swift url
+	"github.com/ncw/rclone/fs"
-var Match = regexp.MustCompile(`^swift://([^/]*)(.*)$`)
+	"github.com/ncw/swift"
+)
 
 // Register with Fs
 func init() {
-	fs.Register(Match, NewFs)
+	fs.Register(&fs.FsInfo{
+		Name:  "swift",
+		NewFs: NewFs,
+		Options: []fs.Option{{
+			Name: "user",
+			Help: "User name to log in.",
+		}, {
+			Name: "key",
+			Help: "API key or password.",
+		}, {
+			Name: "auth",
+			Help: "Authentication URL for server.",
+			Examples: []fs.OptionExample{{
+				Help:  "Rackspace US",
+				Value: "https://auth.api.rackspacecloud.com/v1.0",
+			}, {
+				Help:  "Rackspace UK",
+				Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
+			}, {
+				Help:  "Rackspace v2",
+				Value: "https://identity.api.rackspacecloud.com/v2.0",
+			}, {
+				Help:  "Memset Memstore UK",
+				Value: "https://auth.storage.memset.com/v1.0",
+			}, {
+				Help:  "Memset Memstore UK v2",
+				Value: "https://auth.storage.memset.com/v2.0",
+			}},
+		},
+		// snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
+		},
+	})
 }
 
 // FsSwift represents a remote swift server
@@ -44,48 +71,44 @@ type FsObjectSwift struct {
 
 // ------------------------------------------------------------
 
-// Globals
-var (
-	// Flags
-	// FIXME make these part of swift so we get a standard set of flags?
-	authUrl  = flag.String("swift-auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
-	userName = flag.String("swift-user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
-	apiKey   = flag.String("swift-key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
-	snet     = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
-)
-
 // String converts this FsSwift to a string
 func (f *FsSwift) String() string {
 	return fmt.Sprintf("Swift container %s", f.container)
 }
 
+// Pattern to match a swift path
+var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
+
 // parseParse parses a swift 'url'
 func parsePath(path string) (container, directory string, err error) {
-	parts := Match.FindAllStringSubmatch(path, -1)
+	parts := matcher.FindStringSubmatch(path)
-	if len(parts) != 1 || len(parts[0]) != 3 {
+	if parts == nil {
-		err = fmt.Errorf("Couldn't parse swift url %q", path)
+		err = fmt.Errorf("Couldn't find container in swift path %q", path)
 	} else {
-		container, directory = parts[0][1], parts[0][2]
+		container, directory = parts[1], parts[2]
 		directory = strings.Trim(directory, "/")
 	}
 	return
 }
 
 // swiftConnection makes a connection to swift
-func swiftConnection() (*swift.Connection, error) {
+func swiftConnection(name string) (*swift.Connection, error) {
-	if *userName == "" {
+	userName := fs.ConfigFile.MustValue(name, "user")
-		return nil, errors.New("Need -user or environmental variable ST_USER")
+	if userName == "" {
+		return nil, errors.New("user not found")
 	}
-	if *apiKey == "" {
+	apiKey := fs.ConfigFile.MustValue(name, "key")
-		return nil, errors.New("Need -key or environmental variable ST_KEY")
+	if apiKey == "" {
+		return nil, errors.New("key not found")
 	}
-	if *authUrl == "" {
+	authUrl := fs.ConfigFile.MustValue(name, "auth")
-		return nil, errors.New("Need -auth or environmental variable ST_AUTH")
+	if authUrl == "" {
+		return nil, errors.New("auth not found")
 	}
 	c := &swift.Connection{
-		UserName: *userName,
+		UserName: userName,
-		ApiKey:   *apiKey,
+		ApiKey:   apiKey,
-		AuthUrl:  *authUrl,
+		AuthUrl:  authUrl,
 	}
 	err := c.Authenticate()
 	if err != nil {
@@ -95,7 +118,7 @@ func swiftConnection() (*swift.Connection, error) {
 }
 
 // NewFs contstructs an FsSwift from the path, container:path
-func NewFs(path string) (fs.Fs, error) {
+func NewFs(name, path string) (fs.Fs, error) {
 	container, directory, err := parsePath(path)
 	if err != nil {
 		return nil, err
@@ -103,7 +126,7 @@ func NewFs(path string) (fs.Fs, error) {
 	if directory != "" {
 		return nil, fmt.Errorf("Directories not supported yet in %q", path)
 	}
-	c, err := swiftConnection()
+	c, err := swiftConnection(name)
 	if err != nil {
 		return nil, err
 	}