github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-22 13:03:02 +01:00
Code Releases Activity

Remove filesystem flags and put in config file with editor

Browse Source
This commit is contained in:
Nick Craig-Wood 2014-03-15 16:06:11 +00:00
parent 8fd43a52e7
commit 0a108832e2
8 changed files with 665 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
test-env*
_junk/
rclone
upload

70
drive/fs.go
View File

@ -22,30 +22,52 @@ package drive
// * files with / in name
import (
"code.google.com/p/goauth2/oauth"
"code.google.com/p/google-api-go-client/drive/v2"
"errors"
"flag"
"fmt"
"github.com/ncw/rclone/fs"
"io"
"log"
"mime"
"net/http"
"os"
"path"
"regexp"
"strings"
"sync"
"time"
)
// Pattern to match a drive url
var Match = regexp.MustCompile(`^drive://(.*)$`)
"code.google.com/p/goauth2/oauth"
"code.google.com/p/google-api-go-client/drive/v2"
"github.com/ncw/rclone/fs"
)
// Register with Fs
func init() {
fs.Register(Match, NewFs)
fs.Register(&fs.FsInfo{
Name: "drive",
NewFs: NewFs,
Options: []fs.Option{{
Name: "client_id",
Help: "Google Application Client Id.",
Examples: []fs.OptionExample{{
Value: "202264815644.apps.googleusercontent.com",
Help: "rclone's client id - use this or your own if you want",
}},
}, {
Name: "client_secret",
Help: "Google Application Client Secret.",
Examples: []fs.OptionExample{{
Value: "X4Z3ca8xfWDb1Voo-F9a7ZxJ",
Help: "rclone's client secret - use this or your own if you want",
}},
}, {
Name: "token_file",
Help: "Path to store token file.",
Examples: []fs.OptionExample{{
Value: path.Join(fs.HomeDir, ".gdrive-token-file"),
Help: "Suggested path for token file",
}},
}},
})
}
// FsDrive represents a remote drive server
@ -128,9 +150,6 @@ const (
// Globals
var (
// Flags
driveClientId = flag.String("drive-client-id", os.Getenv("GDRIVE_CLIENT_ID"), "Auth URL for server. Defaults to environment var GDRIVE_CLIENT_ID.")
driveClientSecret = flag.String("drive-client-secret", os.Getenv("GDRIVE_CLIENT_SECRET"), "User name. Defaults to environment var GDRIVE_CLIENT_SECRET.")
driveTokenFile = flag.String("drive-token-file", os.Getenv("GDRIVE_TOKEN_FILE"), "API key (password). Defaults to environment var GDRIVE_TOKEN_FILE.")
driveAuthCode = flag.String("drive-auth-code", "", "Pass in when requested to make the drive token file.")
driveFullList = flag.Bool("drive-full-list", true, "Use a full listing for directory list. More data but usually quicker.")
)
@ -142,13 +161,7 @@ func (f *FsDrive) String() string {
// parseParse parses a drive 'url'
func parseDrivePath(path string) (root string, err error) {
parts := Match.FindAllStringSubmatch(path, -1)
if len(parts) != 1 || len(parts[0]) != 2 {
err = fmt.Errorf("Couldn't parse drive url %q", path)
} else {
root = parts[0][1]
root = strings.Trim(root, "/")
}
return
}
@ -222,26 +235,29 @@ func MakeNewToken(t *oauth.Transport) error {
}
// NewFs contstructs an FsDrive from the path, container:path
func NewFs(path string) (fs.Fs, error) {
if *driveClientId == "" {
return nil, errors.New("Need -drive-client-id or environmental variable GDRIVE_CLIENT_ID")
func NewFs(name, path string) (fs.Fs, error) {
clientId := fs.ConfigFile.MustValue(name, "client_id")
if clientId == "" {
return nil, errors.New("client_id not found")
}
if *driveClientSecret == "" {
return nil, errors.New("Need -drive-client-secret or environmental variable GDRIVE_CLIENT_SECRET")
clientSecret := fs.ConfigFile.MustValue(name, "client_secret")
if clientSecret == "" {
return nil, errors.New("client_secret not found")
}
if *driveTokenFile == "" {
return nil, errors.New("Need -drive-token-file or environmental variable GDRIVE_TOKEN_FILE")
tokenFile := fs.ConfigFile.MustValue(name, "token_file")
if tokenFile == "" {
return nil, errors.New("token-file not found")
}
// Settings for authorization.
var driveConfig = &oauth.Config{
ClientId: *driveClientId,
ClientSecret: *driveClientSecret,
ClientId: clientId,
ClientSecret: clientSecret,
Scope: "https://www.googleapis.com/auth/drive",
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
TokenCache: oauth.CacheFile(*driveTokenFile),
TokenCache: oauth.CacheFile(tokenFile),
}
root, err := parseDrivePath(path)

260
fs/config.go Normal file
View File

@ -0,0 +1,260 @@
// Read and write the config file
package fs
import (
"bufio"
"flag"
"fmt"
"log"
"os"
"os/user"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/Unknwon/goconfig"
)
const (
configFileName = ".rclone.conf"
)
// Global
var (
// Config file
ConfigFile *goconfig.ConfigFile
// Config file path
ConfigPath string
// Global config
Config = &ConfigInfo{}
// Home directory
HomeDir string
// Flags
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
)
// Filesystem config options
type ConfigInfo struct {
Verbose bool
Quiet bool
ModifyWindow time.Duration
Checkers int
Transfers int
}
// Loads the config file
func LoadConfig() {
// Read some flags if set
//
// FIXME read these from the config file too
Config.Verbose = *verbose
Config.Quiet = *quiet
Config.ModifyWindow = *modifyWindow
Config.Checkers = *checkers
Config.Transfers = *transfers
// Find users home directory
usr, err := user.Current()
if err != nil {
log.Printf("Couldn't find home directory: %v", err)
return
}
HomeDir = usr.HomeDir
ConfigPath = path.Join(HomeDir, configFileName)
// Load configuration file.
ConfigFile, err = goconfig.LoadConfigFile(ConfigPath)
if err != nil {
log.Printf("Failed to load config file %v - using defaults", ConfigPath)
}
}
// Save configuration file.
func SaveConfig() {
err := goconfig.SaveConfigFile(ConfigFile, ConfigPath)
if err != nil {
log.Fatalf("Failed to save config file: %v", err)
}
}
// Show an overview of the config file
func ShowConfig() {
remotes := ConfigFile.GetSectionList()
sort.Strings(remotes)
fmt.Printf("%-20s %s\n", "Name", "Type")
fmt.Printf("%-20s %s\n", "====", "====")
for _, remote := range remotes {
fmt.Printf("%-20s %s\n", remote, ConfigFile.MustValue(remote, "type"))
}
}
// ChooseRemote chooses a remote name
func ChooseRemote() string {
remotes := ConfigFile.GetSectionList()
sort.Strings(remotes)
return Choose("remote", remotes, nil, false)
}
// Read some input
func ReadLine() string {
buf := bufio.NewReader(os.Stdin)
line, err := buf.ReadString('\n')
if err != nil {
log.Fatalf("Failed to read line: %v", err)
}
return strings.TrimSpace(line)
}
// Command - choose one
func Command(commands []string) int {
opts := []string{}
for _, text := range commands {
fmt.Printf("%c) %s\n", text[0], text[1:])
opts = append(opts, text[:1])
}
optString := strings.Join(opts, "")
optHelp := strings.Join(opts, "/")
for {
fmt.Printf("%s> ", optHelp)
result := strings.ToLower(ReadLine())
if len(result) != 1 {
continue
}
i := strings.IndexByte(optString, result[0])
if i >= 0 {
return i
}
}
}
// Choose one of the defaults or type a new string if newOk is set
func Choose(what string, defaults, help []string, newOk bool) string {
fmt.Printf("Choose a number from below")
if newOk {
fmt.Printf(", or type in your own value")
}
fmt.Println()
for i, text := range defaults {
if help != nil {
parts := strings.Split(help[i], "\n")
for _, part := range parts {
fmt.Printf(" * %s\n", part)
}
}
fmt.Printf("%2d) %s\n", i+1, text)
}
for {
fmt.Printf("%s> ", what)
result := ReadLine()
i, err := strconv.Atoi(result)
if err != nil {
if newOk {
return result
}
continue
}
if i >= 1 && i <= len(defaults) {
return defaults[i-1]
}
}
}
// Show the contents of the remote
func ShowRemote(name string) {
fmt.Printf("--------------------\n")
fmt.Printf("[%s]\n", name)
for _, key := range ConfigFile.GetKeyList(name) {
fmt.Printf("%s = %s\n", key, ConfigFile.MustValue(name, key))
}
fmt.Printf("--------------------\n")
}
// Print the contents of the remote and ask if it is OK
func OkRemote(name string) bool {
ShowRemote(name)
switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i {
case 0:
return true
case 1:
return false
case 2:
ConfigFile.DeleteSection(name)
return true
default:
log.Printf("Bad choice %d", i)
}
return false
}
// Make a new remote
func NewRemote(name string) {
fmt.Printf("What type of source is it?\n")
types := []string{}
for _, item := range fsRegistry {
types = append(types, item.Name)
}
newType := Choose("type", types, nil, false)
ConfigFile.SetValue(name, "type", newType)
fs, err := Find(newType)
if err != nil {
log.Fatalf("Failed to find fs: %v", err)
}
for _, option := range fs.Options {
ConfigFile.SetValue(name, option.Name, option.Choose())
}
if OkRemote(name) {
SaveConfig()
return
}
EditRemote(name)
}
// Edit a remote
func EditRemote(name string) {
ShowRemote(name)
fmt.Printf("Edit remote\n")
for {
for _, key := range ConfigFile.GetKeyList(name) {
value := ConfigFile.MustValue(name, key)
fmt.Printf("Press enter to accept current value, or type in a new one\n")
fmt.Printf("%s = %s>", key, value)
newValue := ReadLine()
if newValue != "" {
ConfigFile.SetValue(name, key, newValue)
}
}
if OkRemote(name) {
break
}
}
SaveConfig()
}
// Edit the config file interactively
func EditConfig() {
for {
fmt.Printf("Current remotes:\n\n")
ShowConfig()
fmt.Printf("\n")
switch i := Command([]string{"eEdit existing remote", "nNew remote", "dDelete remote", "qQuit config"}); i {
case 0:
name := ChooseRemote()
EditRemote(name)
case 1:
fmt.Printf("name> ")
name := ReadLine()
NewRemote(name)
case 2:
name := ChooseRemote()
ConfigFile.DeleteSection(name)
case 3:
return
}
}
}

97
fs/fs.go
View File

@ -12,41 +12,52 @@ import (
// Globals
var (
// Global config
Config = &ConfigInfo{}
// Filesystem registry
fsRegistry []registryItem
fsRegistry []*FsInfo
)
// Filesystem config options
type ConfigInfo struct {
Verbose bool
Quiet bool
ModifyWindow time.Duration
Checkers int
Transfers int
// Filesystem info
type FsInfo struct {
Name string // name of this fs
NewFs func(string, string) (Fs, error) // create a new file system
Options []Option
}
// Filesystem registry item
type registryItem struct {
match *regexp.Regexp // if this matches then can call newFs
newFs func(string) (Fs, error) // create a new file system
// An options for a Fs
type Option struct {
Name string
Help string
Optional bool
Examples []OptionExample
}
// An example for an option
type OptionExample struct {
Value string
Help string
}
// Choose an option
func (o *Option) Choose() string {
fmt.Println(o.Help)
if len(o.Examples) > 0 {
var values []string
var help []string
for _, example := range o.Examples {
values = append(values, example.Value)
help = append(help, example.Help)
}
return Choose(o.Name, values, help, true)
}
fmt.Printf("%s> ", o.Name)
return ReadLine()
}
// Register a filesystem
//
// If a path matches with match then can call newFs on it
//
// Pass with match nil goes last and matches everything (used by local fs)
//
// Fs modules should use this in an init() function
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
// Keep one nil match at the end
last := len(fsRegistry) - 1
if last >= 1 && fsRegistry[last-1].match == nil {
fsRegistry[last], fsRegistry[last-1] = fsRegistry[last-1], fsRegistry[last]
}
func Register(info *FsInfo) {
fsRegistry = append(fsRegistry, info)
}
// A Filesystem, describes the local filesystem and the remote object store
@ -136,16 +147,42 @@ type Dir struct {
// A channel of Dir objects
type DirChan chan *Dir
// Pattern to match a url
var matcher = regexp.MustCompile(`^([\w_-]+)://(.*)$`)
// Finds a FsInfo object for the name passed in
//
// Services are looked up in the config file
func Find(name string) (*FsInfo, error) {
for _, item := range fsRegistry {
if item.Name == name {
return item, nil
}
}
return nil, fmt.Errorf("Didn't find filing system for %q", name)
}
// NewFs makes a new Fs object from the path
//
// FIXME make more generic
// The path is of the form service://path
//
// Services are looked up in the config file
func NewFs(path string) (Fs, error) {
for _, item := range fsRegistry {
if item.match == nil || item.match.MatchString(path) {
return item.newFs(path)
parts := matcher.FindStringSubmatch(path)
fsName, configName, fsPath := "local", "local", path
if parts != nil {
configName, fsPath = parts[1], parts[2]
var err error
fsName, err = ConfigFile.GetValue(configName, "type")
if err != nil {
return nil, fmt.Errorf("Didn't find section in config file for %q", configName)
}
}
panic("Not found") // FIXME
fs, err := Find(fsName)
if err != nil {
return nil, err
}
return fs.NewFs(configName, fsPath)
}
// Write debuging output for this Object

10
local/fs.go
View File

@ -4,7 +4,6 @@ package local
import (
"crypto/md5"
"fmt"
"github.com/ncw/rclone/fs"
"io"
"io/ioutil"
"log"
@ -13,11 +12,16 @@ import (
"path/filepath"
"sync"
"time"
"github.com/ncw/rclone/fs"
)
// Register with Fs
func init() {
fs.Register(nil, NewFs)
fs.Register(&fs.FsInfo{
Name: "local",
NewFs: NewFs,
})
}
// FsLocal represents a local filesystem rooted at root
@ -37,7 +41,7 @@ type FsObjectLocal struct {
// ------------------------------------------------------------
// NewFs contstructs an FsLocal from the path
func NewFs(root string) (fs.Fs, error) {
func NewFs(name, root string) (fs.Fs, error) {
root = path.Clean(root)
f := &FsLocal{root: root}
return f, nil

216
rclone.go
View File

@ -6,7 +6,6 @@ package main
import (
"flag"
"fmt"
"github.com/ncw/rclone/fs"
"log"
"os"
"runtime"
@ -14,6 +13,8 @@ import (
"strings"
"sync"
"time"
"github.com/ncw/rclone/fs"
// Active file systems
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/local"
@ -25,13 +26,8 @@ import (
var (
// Flags
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
dry_run = flag.Bool("dry-run", false, "Do a trial run with no permanent changes")
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
transfers = flag.Int("transfers", 4, "Number of file transfers to run in parallel.")
statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
)
// A pair of fs.Objects
@ -105,17 +101,17 @@ func CopyFs(fdst, fsrc fs.Fs) {
}
to_be_checked := fsrc.List()
to_be_uploaded := make(fs.ObjectsChan, *transfers)
to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
var checkerWg sync.WaitGroup
checkerWg.Add(*checkers)
for i := 0; i < *checkers; i++ {
checkerWg.Add(fs.Config.Checkers)
for i := 0; i < fs.Config.Checkers; i++ {
go Checker(to_be_checked, to_be_uploaded, fdst, &checkerWg)
}
var copierWg sync.WaitGroup
copierWg.Add(*transfers)
for i := 0; i < *transfers; i++ {
copierWg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go Copier(to_be_uploaded, fdst, &copierWg)
}
@ -129,8 +125,8 @@ func CopyFs(fdst, fsrc fs.Fs) {
// Delete all the files passed in the channel
func DeleteFiles(to_be_deleted fs.ObjectsChan) {
var wg sync.WaitGroup
wg.Add(*transfers)
for i := 0; i < *transfers; i++ {
wg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go func() {
defer wg.Done()
for dst := range to_be_deleted {
@ -173,18 +169,18 @@ func Sync(fdst, fsrc fs.Fs) {
}
// Read source files checking them off against dest files
to_be_checked := make(PairFsObjectsChan, *transfers)
to_be_uploaded := make(fs.ObjectsChan, *transfers)
to_be_checked := make(PairFsObjectsChan, fs.Config.Transfers)
to_be_uploaded := make(fs.ObjectsChan, fs.Config.Transfers)
var checkerWg sync.WaitGroup
checkerWg.Add(*checkers)
for i := 0; i < *checkers; i++ {
checkerWg.Add(fs.Config.Checkers)
for i := 0; i < fs.Config.Checkers; i++ {
go PairChecker(to_be_checked, to_be_uploaded, &checkerWg)
}
var copierWg sync.WaitGroup
copierWg.Add(*transfers)
for i := 0; i < *transfers; i++ {
copierWg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go Copier(to_be_uploaded, fdst, &copierWg)
}
@ -215,7 +211,7 @@ func Sync(fdst, fsrc fs.Fs) {
}
// Delete the spare files
toDelete := make(fs.ObjectsChan, *transfers)
toDelete := make(fs.ObjectsChan, fs.Config.Transfers)
go func() {
for _, fs := range delFiles {
toDelete <- fs
@ -262,7 +258,7 @@ func Check(fdst, fsrc fs.Fs) {
log.Printf(remote)
}
checks := make(chan []fs.Object, *transfers)
checks := make(chan []fs.Object, fs.Config.Transfers)
go func() {
for _, check := range commonFiles {
checks <- check
@ -271,8 +267,8 @@ func Check(fdst, fsrc fs.Fs) {
}()
var checkerWg sync.WaitGroup
checkerWg.Add(*checkers)
for i := 0; i < *checkers; i++ {
checkerWg.Add(fs.Config.Checkers)
for i := 0; i < fs.Config.Checkers; i++ {
go func() {
defer checkerWg.Done()
for check := range checks {
@ -309,8 +305,8 @@ func Check(fdst, fsrc fs.Fs) {
func List(f, _ fs.Fs) {
in := f.List()
var wg sync.WaitGroup
wg.Add(*checkers)
for i := 0; i < *checkers; i++ {
wg.Add(fs.Config.Checkers)
for i := 0; i < fs.Config.Checkers; i++ {
go func() {
defer wg.Done()
for o := range in {
@ -370,117 +366,128 @@ func purge(fdst, fsrc fs.Fs) {
}
}
// Edits the config file
func EditConfig(fdst, fsrc fs.Fs) {
fs.EditConfig()
}
type Command struct {
name string
help string
run func(fdst, fsrc fs.Fs)
minArgs, maxArgs int
Name string
Help string
ArgsHelp string
Run func(fdst, fsrc fs.Fs)
MinArgs int
MaxArgs int
NoStats bool
}
// checkArgs checks there are enough arguments and prints a message if not
func (cmd *Command) checkArgs(args []string) {
if len(args) < cmd.minArgs {
if len(args) < cmd.MinArgs {
syntaxError()
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.name, cmd.minArgs)
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name, cmd.MinArgs)
os.Exit(1)
} else if len(args) > cmd.maxArgs {
} else if len(args) > cmd.MaxArgs {
syntaxError()
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.name, cmd.maxArgs)
fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name, cmd.MaxArgs)
os.Exit(1)
}
}
var Commands = []Command{
{
"copy",
`<source> <destination>
Name: "copy",
ArgsHelp: "source://path dest://path",
Help: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing first by modification time then by
MD5SUM. Doesn't delete files from the destination.
`,
CopyFs,
2, 2,
MD5SUM. Doesn't delete files from the destination.`,
Run: CopyFs,
MinArgs: 2,
MaxArgs: 2,
},
{
"sync",
`<source> <destination>
Name: "sync",
ArgsHelp: "source://path dest://path",
Help: `
Sync the source to the destination. Doesn't transfer
unchanged files, testing first by modification time then by
MD5SUM. Deletes any files that exist in source that don't
exist in destination. Since this can cause data loss, test
first with the -dry-run flag.`,
Sync,
2, 2,
Run: Sync,
MinArgs: 2,
MaxArgs: 2,
},
{
"ls",
`[<path>]
Name: "ls",
ArgsHelp: "[remote://path]",
Help: `
List all the objects in the the path.`,
List,
1, 1,
Run: List,
MinArgs: 1,
MaxArgs: 1,
},
{
"lsd",
`[<path>]
Name: "lsd",
ArgsHelp: "[remote://path]",
Help: `
List all directoryes/objects/buckets in the the path.`,
ListDir,
1, 1,
Run: ListDir,
MinArgs: 1,
MaxArgs: 1,
},
{
"mkdir",
`<path>
Name: "mkdir",
ArgsHelp: "remote://path",
Help: `
Make the path if it doesn't already exist`,
mkdir,
1, 1,
Run: mkdir,
MinArgs: 1,
MaxArgs: 1,
},
{
"rmdir",
`<path>
Name: "rmdir",
ArgsHelp: "remote://path",
Help: `
Remove the path. Note that you can't remove a path with
objects in it, use purge for that.`,
rmdir,
1, 1,
Run: rmdir,
MinArgs: 1,
MaxArgs: 1,
},
{
"purge",
`<path>
Name: "purge",
ArgsHelp: "remote://path",
Help: `
Remove the path and all of its contents.`,
purge,
1, 1,
Run: purge,
MinArgs: 1,
MaxArgs: 1,
},
{
"check",
`<source> <destination>
Name: "check",
ArgsHelp: "source://path dest://path",
Help: `
Checks the files in the source and destination match. It
compares sizes and MD5SUMs and prints a report of files which
don't match. It doesn't alter the source or destination.`,
Check,
2, 2,
Run: Check,
MinArgs: 2,
MaxArgs: 2,
},
{
"help",
`
Name: "config",
Help: `
Enter an interactive configuration session.`,
Run: EditConfig,
NoStats: true,
},
{
Name: "help",
Help: `
This help.`,
nil,
0, 0,
NoStats: true,
},
}
@ -495,7 +502,8 @@ Subcommands:
`)
for i := range Commands {
cmd := &Commands[i]
fmt.Fprintf(os.Stderr, " %s: %s\n\n", cmd.name, cmd.help)
fmt.Fprintf(os.Stderr, " %s %s\n", cmd.Name, cmd.ArgsHelp)
fmt.Fprintf(os.Stderr, "%s\n\n", cmd.Help)
}
fmt.Fprintf(os.Stderr, "Options:\n")
@ -517,13 +525,7 @@ func main() {
flag.Parse()
args := flag.Args()
runtime.GOMAXPROCS(runtime.NumCPU())
// Pass on some flags to fs.Config
fs.Config.Verbose = *verbose
fs.Config.Quiet = *quiet
fs.Config.ModifyWindow = *modifyWindow
fs.Config.Checkers = *checkers
fs.Config.Transfers = *transfers
fs.LoadConfig()
// Setup profiling if desired
if *cpuprofile != "" {
@ -548,10 +550,10 @@ func main() {
for i := range Commands {
command := &Commands[i]
// exact command name found - use that
if command.name == cmd {
if command.Name == cmd {
found = command
break
} else if strings.HasPrefix(command.name, cmd) {
} else if strings.HasPrefix(command.Name, cmd) {
if found != nil {
fs.Stats.Error()
log.Fatalf("Not unique - matches multiple commands %q", cmd)
@ -572,14 +574,14 @@ func main() {
fdst, err = fs.NewFs(args[0])
if err != nil {
fs.Stats.Error()
log.Fatal("Failed to create file system: ", err)
log.Fatalf("Failed to create file system for %q: %v", args[0], err)
}
}
if len(args) >= 2 {
fsrc, err = fs.NewFs(args[1])
if err != nil {
fs.Stats.Error()
log.Fatal("Failed to create destination file system: ", err)
log.Fatalf("Failed to create destination file system for %q: %v", args[1], err)
}
fsrc, fdst = fdst, fsrc
}
@ -599,9 +601,12 @@ func main() {
fs.Config.ModifyWindow = precision
}
}
if fs.Config.Verbose {
log.Printf("Modify window is %s\n", fs.Config.ModifyWindow)
}
// Print the stats every statsInterval
if !found.NoStats {
go func() {
ch := time.Tick(*statsInterval)
for {
@ -609,12 +614,17 @@ func main() {
fs.Stats.Log()
}
}()
}
// Run the actual command
if found.run != nil {
found.run(fdst, fsrc)
if found.Run != nil {
found.Run(fdst, fsrc)
if !found.NoStats {
fmt.Println(fs.Stats)
}
if fs.Config.Verbose {
log.Printf("*** Go routines at exit %d\n", runtime.NumGoroutine())
}
if fs.Stats.Errored() {
os.Exit(1)
}

140
s3/fs.go
View File

@ -5,30 +5,99 @@ package s3
import (
"errors"
"flag"
"fmt"
"github.com/ncw/goamz/aws"
"github.com/ncw/goamz/s3"
"github.com/ncw/rclone/fs"
"github.com/ncw/swift"
"io"
"log"
"mime"
"net/http"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
)
// Pattern to match a s3 url
var Match = regexp.MustCompile(`^s3://([^/]*)(.*)$`)
"github.com/ncw/goamz/aws"
"github.com/ncw/goamz/s3"
"github.com/ncw/rclone/fs"
"github.com/ncw/swift"
)
// Register with Fs
func init() {
fs.Register(Match, NewFs)
fs.Register(&fs.FsInfo{
Name: "s3",
NewFs: NewFs,
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
Options: []fs.Option{{
Name: "access_key_id",
Help: "AWS Access Key ID.",
}, {
Name: "secret_access_key",
Help: "AWS Secret Access Key (password). ",
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.",
Examples: []fs.OptionExample{{
Value: "https://s3.amazonaws.com/",
Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.",
}, {
Value: "https://s3-external-1.amazonaws.com",
Help: "US Region, Northern Virginia only.\nLeave location constraint empty.",
}, {
Value: "https://s3-us-west-2.amazonaws.com",
Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.",
}, {
Value: "https://s3-us-west-1.amazonaws.com",
Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.",
}, {
Value: "https://s3-eu-west-1.amazonaws.com",
Help: "EU (Ireland) Region Region\nNeeds location constraint EU or eu-west-1.",
}, {
Value: "https://s3-ap-southeast-1.amazonaws.com",
Help: "Asia Pacific (Singapore) Region\nNeeds location constraint ap-southeast-1.",
}, {
Value: "https://s3-ap-southeast-2.amazonaws.com",
Help: "Asia Pacific (Sydney) Region\nNeeds location constraint .",
}, {
Value: "https://s3-ap-northeast-1.amazonaws.com",
Help: "Asia Pacific (Tokyo) Region\nNeeds location constraint ap-northeast-1.",
}, {
Value: "https://s3-sa-east-1.amazonaws.com",
Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
}},
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Endpoint.",
Examples: []fs.OptionExample{{
Value: "",
Help: "Empty for US Region, Northern Virginia or Pacific Northwest.",
}, {
Value: "us-west-2",
Help: "US West (Oregon) Region.",
}, {
Value: "us-west-1",
Help: "US West (Northern California) Region.",
}, {
Value: "eu-west-1",
Help: "EU (Ireland) Region.",
}, {
Value: "EU",
Help: "EU Region.",
}, {
Value: "ap-southeast-1",
Help: "Asia Pacific (Singapore) Region.",
}, {
Value: "ap-southeast-2",
Help: "Asia Pacific (Sydney) Region.",
}, {
Value: "ap-northeast-1",
Help: "Asia Pacific (Tokyo) Region.",
}, {
Value: "sa-east-1",
Help: "South America (Sao Paulo) Region.",
}},
}},
})
}
// Constants
@ -60,57 +129,54 @@ type FsObjectS3 struct {
// ------------------------------------------------------------
// Globals
var (
// Flags
awsAccessKeyId = flag.String("aws-access-key-id", os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key ID. Defaults to environment var AWS_ACCESS_KEY_ID.")
awsSecretAccessKey = flag.String("aws-secret-access-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Access Key (password). Defaults to environment var AWS_SECRET_ACCESS_KEY.")
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
s3Endpoint = flag.String("s3-endpoint", os.Getenv("S3_ENDPOINT"), "S3 Endpoint. Defaults to environment var S3_ENDPOINT then https://s3.amazonaws.com/.")
s3LocationConstraint = flag.String("s3-location-constraint", os.Getenv("S3_LOCATION_CONSTRAINT"), "Location constraint for creating buckets only. Defaults to environment var S3_LOCATION_CONSTRAINT.")
)
// String converts this FsS3 to a string
func (f *FsS3) String() string {
return fmt.Sprintf("S3 bucket %s", f.bucket)
}
// Pattern to match a s3 path
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
// parseParse parses a s3 'url'
func s3ParsePath(path string) (bucket, directory string, err error) {
parts := Match.FindAllStringSubmatch(path, -1)
if len(parts) != 1 || len(parts[0]) != 3 {
err = fmt.Errorf("Couldn't parse s3 url %q", path)
parts := matcher.FindStringSubmatch(path)
if parts == nil {
err = fmt.Errorf("Couldn't parse bucket out of s3 path %q", path)
} else {
bucket, directory = parts[0][1], parts[0][2]
bucket, directory = parts[1], parts[2]
directory = strings.Trim(directory, "/")
}
return
}
// s3Connection makes a connection to s3
func s3Connection() (*s3.S3, error) {
func s3Connection(name string) (*s3.S3, error) {
// Make the auth
if *awsAccessKeyId == "" {
return nil, errors.New("Need -aws-access-key-id or environmental variable AWS_ACCESS_KEY_ID")
accessKeyId := fs.ConfigFile.MustValue(name, "access_key_id")
if accessKeyId == "" {
return nil, errors.New("access_key_id not found")
}
if *awsSecretAccessKey == "" {
return nil, errors.New("Need -aws-secret-access-key or environmental variable AWS_SECRET_ACCESS_KEY")
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
if secretAccessKey == "" {
return nil, errors.New("secret_access_key not found")
}
auth := aws.Auth{AccessKey: *awsAccessKeyId, SecretKey: *awsSecretAccessKey}
auth := aws.Auth{AccessKey: accessKeyId, SecretKey: secretAccessKey}
// FIXME look through all the regions by name and use one of them if found
// Synthesize the region
if *s3Endpoint == "" {
*s3Endpoint = "https://s3.amazonaws.com/"
s3Endpoint := fs.ConfigFile.MustValue(name, "endpoint")
if s3Endpoint == "" {
s3Endpoint = "https://s3.amazonaws.com/"
}
region := aws.Region{
Name: "s3",
S3Endpoint: *s3Endpoint,
S3Endpoint: s3Endpoint,
S3LocationConstraint: false,
}
if *s3LocationConstraint != "" {
region.Name = *s3LocationConstraint
s3LocationConstraint := fs.ConfigFile.MustValue(name, "location_constraint")
if s3LocationConstraint != "" {
region.Name = s3LocationConstraint
region.S3LocationConstraint = true
}
@ -119,7 +185,7 @@ func s3Connection() (*s3.S3, error) {
}
// NewFsS3 contstructs an FsS3 from the path, bucket:path
func NewFs(path string) (fs.Fs, error) {
func NewFs(name, path string) (fs.Fs, error) {
bucket, directory, err := s3ParsePath(path)
if err != nil {
return nil, err
@ -127,7 +193,7 @@ func NewFs(path string) (fs.Fs, error) {
if directory != "" {
return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory)
}
c, err := s3Connection()
c, err := s3Connection(name)
if err != nil {
return nil, err
}

91
swift/fs.go
View File

@ -5,24 +5,51 @@ package swift
import (
"errors"
"flag"
"fmt"
"github.com/ncw/rclone/fs"
"github.com/ncw/swift"
"io"
"log"
"os"
"regexp"
"strings"
"time"
)
// Pattern to match a swift url
var Match = regexp.MustCompile(`^swift://([^/]*)(.*)$`)
"github.com/ncw/rclone/fs"
"github.com/ncw/swift"
)
// Register with Fs
func init() {
fs.Register(Match, NewFs)
fs.Register(&fs.FsInfo{
Name: "swift",
NewFs: NewFs,
Options: []fs.Option{{
Name: "user",
Help: "User name to log in.",
}, {
Name: "key",
Help: "API key or password.",
}, {
Name: "auth",
Help: "Authentication URL for server.",
Examples: []fs.OptionExample{{
Help: "Rackspace US",
Value: "https://auth.api.rackspacecloud.com/v1.0",
}, {
Help: "Rackspace UK",
Value: "https://lon.auth.api.rackspacecloud.com/v1.0",
}, {
Help: "Rackspace v2",
Value: "https://identity.api.rackspacecloud.com/v2.0",
}, {
Help: "Memset Memstore UK",
Value: "https://auth.storage.memset.com/v1.0",
}, {
Help: "Memset Memstore UK v2",
Value: "https://auth.storage.memset.com/v2.0",
}},
},
// snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
},
})
}
// FsSwift represents a remote swift server
@ -44,48 +71,44 @@ type FsObjectSwift struct {
// ------------------------------------------------------------
// Globals
var (
// Flags
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("swift-auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
userName = flag.String("swift-user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("swift-key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
)
// String converts this FsSwift to a string
func (f *FsSwift) String() string {
return fmt.Sprintf("Swift container %s", f.container)
}
// Pattern to match a swift path
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
// parseParse parses a swift 'url'
func parsePath(path string) (container, directory string, err error) {
parts := Match.FindAllStringSubmatch(path, -1)
if len(parts) != 1 || len(parts[0]) != 3 {
err = fmt.Errorf("Couldn't parse swift url %q", path)
parts := matcher.FindStringSubmatch(path)
if parts == nil {
err = fmt.Errorf("Couldn't find container in swift path %q", path)
} else {
container, directory = parts[0][1], parts[0][2]
container, directory = parts[1], parts[2]
directory = strings.Trim(directory, "/")
}
return
}
// swiftConnection makes a connection to swift
func swiftConnection() (*swift.Connection, error) {
if *userName == "" {
return nil, errors.New("Need -user or environmental variable ST_USER")
func swiftConnection(name string) (*swift.Connection, error) {
userName := fs.ConfigFile.MustValue(name, "user")
if userName == "" {
return nil, errors.New("user not found")
}
if *apiKey == "" {
return nil, errors.New("Need -key or environmental variable ST_KEY")
apiKey := fs.ConfigFile.MustValue(name, "key")
if apiKey == "" {
return nil, errors.New("key not found")
}
if *authUrl == "" {
return nil, errors.New("Need -auth or environmental variable ST_AUTH")
authUrl := fs.ConfigFile.MustValue(name, "auth")
if authUrl == "" {
return nil, errors.New("auth not found")
}
c := &swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
UserName: userName,
ApiKey: apiKey,
AuthUrl: authUrl,
}
err := c.Authenticate()
if err != nil {
@ -95,7 +118,7 @@ func swiftConnection() (*swift.Connection, error) {
}
// NewFs contstructs an FsSwift from the path, container:path
func NewFs(path string) (fs.Fs, error) {
func NewFs(name, path string) (fs.Fs, error) {
container, directory, err := parsePath(path)
if err != nil {
return nil, err
@ -103,7 +126,7 @@ func NewFs(path string) (fs.Fs, error) {
if directory != "" {
return nil, fmt.Errorf("Directories not supported yet in %q", path)
}
c, err := swiftConnection()
c, err := swiftConnection(name)
if err != nil {
return nil, err
}