2012-11-18 18:32:31 +01:00
|
|
|
// Sync files and directories to and from swift
|
|
|
|
//
|
|
|
|
// Nick Craig-Wood <nick@craig-wood.com>
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2012-11-28 12:17:31 +01:00
|
|
|
"crypto/md5"
|
2012-11-18 18:32:31 +01:00
|
|
|
"flag"
|
|
|
|
"fmt"
|
2012-11-20 23:40:48 +01:00
|
|
|
"github.com/ncw/swift"
|
2012-11-28 12:17:31 +01:00
|
|
|
"io"
|
2012-11-18 18:32:31 +01:00
|
|
|
"log"
|
|
|
|
"os"
|
2012-11-30 00:07:29 +01:00
|
|
|
"path"
|
2012-11-20 23:40:48 +01:00
|
|
|
"path/filepath"
|
2012-11-29 23:13:58 +01:00
|
|
|
"runtime"
|
2012-11-18 18:32:31 +01:00
|
|
|
"runtime/pprof"
|
2012-11-28 12:17:31 +01:00
|
|
|
"strings"
|
2012-11-29 23:13:58 +01:00
|
|
|
"sync"
|
2012-12-01 11:53:48 +01:00
|
|
|
"time"
|
2012-11-18 18:32:31 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Flags
|
|
|
|
//fileSize = flag.Int64("s", 1E9, "Size of the check files")
|
|
|
|
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
|
|
|
|
//duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
|
|
|
|
//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
|
|
|
|
//logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
|
|
|
|
|
|
|
|
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
|
|
|
|
verbose = flag.Bool("verbose", false, "Print lots more stuff")
|
|
|
|
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
|
|
|
|
// FIXME make these part of swift so we get a standard set of flags?
|
2012-11-29 23:13:58 +01:00
|
|
|
authUrl = flag.String("auth", os.Getenv("ST_AUTH"), "Auth URL for server. Defaults to environment var ST_AUTH.")
|
|
|
|
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
|
|
|
|
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
|
|
|
|
checkers = flag.Int("checkers", 8, "Number of checkers to run in parallel.")
|
|
|
|
uploaders = flag.Int("uploaders", 4, "Number of uploaders to run in parallel.")
|
2012-11-18 18:32:31 +01:00
|
|
|
)
|
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
// A filesystem like object which can either be a remote object or a
|
|
|
|
// local file/directory or both
|
2012-11-20 23:40:48 +01:00
|
|
|
type FsObject struct {
|
2012-12-01 11:53:48 +01:00
|
|
|
remote string // The remote path
|
|
|
|
path string // The local path
|
|
|
|
info os.FileInfo // Interface for file info
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
type FsObjectsChan chan *FsObject
|
|
|
|
|
2012-11-28 12:38:14 +01:00
|
|
|
type FsObjects []FsObject
|
2012-11-20 23:40:48 +01:00
|
|
|
|
2012-11-28 18:12:22 +01:00
|
|
|
// Write debuging output for this FsObject
|
2012-11-29 00:40:09 +01:00
|
|
|
func (fs *FsObject) Debugf(text string, args ...interface{}) {
|
2012-11-28 18:12:22 +01:00
|
|
|
out := fmt.Sprintf(text, args...)
|
2012-12-01 11:53:48 +01:00
|
|
|
log.Printf("%s: %s", fs.remote, out)
|
2012-11-28 18:12:22 +01:00
|
|
|
}
|
|
|
|
|
2012-11-28 12:17:31 +01:00
|
|
|
// md5sum calculates the md5sum of a file returning a lowercase hex string
|
2012-11-28 18:12:22 +01:00
|
|
|
func (fs *FsObject) md5sum() (string, error) {
|
|
|
|
in, err := os.Open(fs.path)
|
2012-11-28 12:17:31 +01:00
|
|
|
if err != nil {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Failed to open: %s", err)
|
2012-11-28 12:17:31 +01:00
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer in.Close() // FIXME ignoring error
|
|
|
|
hash := md5.New()
|
|
|
|
_, err = io.Copy(hash, in)
|
|
|
|
if err != nil {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Failed to read: %s", err)
|
2012-11-28 12:17:31 +01:00
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
|
|
|
}
|
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
// Sets the modification time of the local fs object
|
|
|
|
func (fs *FsObject) SetModTime(modTime time.Time) {
|
2012-12-01 13:05:17 +01:00
|
|
|
err := Chtimes(fs.path, modTime, modTime)
|
2012-12-01 11:53:48 +01:00
|
|
|
if err != nil {
|
|
|
|
fs.Debugf("Failed to set mtime on file: %s", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Checks to see if the remote and local objects are equal by looking
|
|
|
|
// at size, mtime and MD5SUM
|
2012-11-27 23:28:46 +01:00
|
|
|
//
|
2012-12-01 11:53:48 +01:00
|
|
|
// If the remote and local size are different then it is considered to
|
|
|
|
// be not equal.
|
2012-11-28 12:17:31 +01:00
|
|
|
//
|
2012-11-28 18:24:00 +01:00
|
|
|
// If the size is the same and the mtime is the same then it is
|
2012-12-01 11:53:48 +01:00
|
|
|
// considered to be equal. This is the heuristic rsync uses when
|
2012-11-28 18:24:00 +01:00
|
|
|
// not using --checksum.
|
|
|
|
//
|
|
|
|
// If the size is the same and and mtime is different or unreadable
|
|
|
|
// and the MD5SUM is the same then the file is considered to be
|
2012-12-01 11:53:48 +01:00
|
|
|
// equal. In this case the mtime on the object is updated. If
|
|
|
|
// upload is set then the remote object is changed otherwise the local
|
|
|
|
// object.
|
2012-11-28 18:24:00 +01:00
|
|
|
//
|
2012-12-01 11:53:48 +01:00
|
|
|
// Otherwise the file is considered to be not equal including if there
|
|
|
|
// were errors reading info.
|
|
|
|
func (fs *FsObject) Equal(c *swift.Connection, container string, upload bool) bool {
|
|
|
|
// FIXME could pass in an Object here if we have one which
|
|
|
|
// will mean we could do the Size and Hash checks without a
|
|
|
|
// remote call if we wanted
|
|
|
|
obj, h, err := c.Object(container, fs.remote)
|
2012-11-27 23:28:46 +01:00
|
|
|
if err != nil {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Failed to read info: %s", err)
|
2012-12-01 11:53:48 +01:00
|
|
|
return false
|
2012-11-27 23:28:46 +01:00
|
|
|
}
|
|
|
|
if obj.Bytes != fs.info.Size() {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Sizes differ")
|
2012-12-01 11:53:48 +01:00
|
|
|
return false
|
2012-11-27 23:28:46 +01:00
|
|
|
}
|
2012-11-28 18:24:00 +01:00
|
|
|
|
|
|
|
// Size the same so check the mtime
|
2012-11-27 23:28:46 +01:00
|
|
|
m := h.ObjectMetadata()
|
2012-12-01 11:53:48 +01:00
|
|
|
remoteModTime, err := m.GetModTime()
|
2012-11-27 23:28:46 +01:00
|
|
|
if err != nil {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Failed to read mtime: %s", err)
|
2012-12-01 11:53:48 +01:00
|
|
|
} else if !remoteModTime.Equal(fs.info.ModTime()) {
|
2012-11-28 18:24:00 +01:00
|
|
|
fs.Debugf("Modification times differ")
|
|
|
|
} else {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Size and modification time the same")
|
|
|
|
return true
|
2012-11-29 00:40:09 +01:00
|
|
|
}
|
2012-11-28 18:24:00 +01:00
|
|
|
|
|
|
|
// mtime is unreadable or different but size is the same so
|
|
|
|
// check the MD5SUM
|
|
|
|
localMd5, err := fs.md5sum()
|
|
|
|
if err != nil {
|
|
|
|
fs.Debugf("Failed to calculate md5: %s", err)
|
2012-12-01 11:53:48 +01:00
|
|
|
return false
|
2012-11-27 23:28:46 +01:00
|
|
|
}
|
2012-11-28 18:24:00 +01:00
|
|
|
// fs.Debugf("Local MD5 %s", localMd5)
|
|
|
|
// fs.Debugf("Remote MD5 %s", obj.Hash)
|
|
|
|
if localMd5 != strings.ToLower(obj.Hash) {
|
|
|
|
fs.Debugf("Md5sums differ")
|
2012-12-01 11:53:48 +01:00
|
|
|
return false
|
2012-11-27 23:28:46 +01:00
|
|
|
}
|
2012-11-28 18:24:00 +01:00
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
// Size and MD5 the same but mtime different so update the
|
|
|
|
// mtime of the local or remote object here
|
|
|
|
if upload {
|
|
|
|
m.SetModTime(fs.info.ModTime())
|
|
|
|
err = c.ObjectUpdate(container, fs.remote, m.ObjectHeaders())
|
|
|
|
if err != nil {
|
|
|
|
fs.Debugf("Failed to update remote mtime: %s", err)
|
|
|
|
}
|
|
|
|
fs.Debugf("Updated mtime of remote object")
|
|
|
|
} else {
|
|
|
|
fmt.Printf("metadata %q, remoteModTime = %s\n", m, remoteModTime)
|
|
|
|
fs.SetModTime(remoteModTime)
|
|
|
|
fs.Debugf("Updated mtime of local object")
|
2012-11-28 18:42:12 +01:00
|
|
|
}
|
2012-11-29 00:40:09 +01:00
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Size and MD5SUM of local and remote objects identical")
|
|
|
|
return true
|
2012-11-27 23:28:46 +01:00
|
|
|
}
|
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
// Is this object storable
|
2012-12-01 11:53:48 +01:00
|
|
|
func (fs *FsObject) storable() bool {
|
2012-11-20 23:40:48 +01:00
|
|
|
mode := fs.info.Mode()
|
|
|
|
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("Can't transfer non file/directory")
|
2012-11-29 23:13:58 +01:00
|
|
|
return false
|
2012-11-20 23:40:48 +01:00
|
|
|
} else if mode&os.ModeDir != 0 {
|
|
|
|
// Debug?
|
2012-11-28 18:12:22 +01:00
|
|
|
fs.Debugf("FIXME Skipping directory")
|
2012-11-29 23:13:58 +01:00
|
|
|
return false
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
2012-11-29 23:13:58 +01:00
|
|
|
return true
|
|
|
|
}
|
2012-11-20 23:40:48 +01:00
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
// Puts the FsObject into the container
|
|
|
|
func (fs *FsObject) put(c *swift.Connection, container string) {
|
|
|
|
// FIXME content type
|
|
|
|
in, err := os.Open(fs.path)
|
|
|
|
if err != nil {
|
|
|
|
fs.Debugf("Failed to open: %s", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer in.Close()
|
|
|
|
m := swift.Metadata{}
|
|
|
|
m.SetModTime(fs.info.ModTime())
|
2012-12-01 11:53:48 +01:00
|
|
|
_, err = c.ObjectPut(container, fs.remote, in, true, "", "", m.ObjectHeaders())
|
2012-11-29 23:13:58 +01:00
|
|
|
if err != nil {
|
|
|
|
fs.Debugf("Failed to upload: %s", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fs.Debugf("Uploaded")
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
// Stat a FsObject info info
|
|
|
|
func (fs *FsObject) lstat() error {
|
|
|
|
info, err := os.Lstat(fs.path)
|
|
|
|
fs.info = info
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2012-11-30 00:07:29 +01:00
|
|
|
// Return an FsObject from a path
|
|
|
|
//
|
|
|
|
// May return nil if an error occurred
|
|
|
|
func NewFsObject(root, path string) *FsObject {
|
2012-12-01 11:53:48 +01:00
|
|
|
remote, err := filepath.Rel(root, path)
|
2012-11-30 00:07:29 +01:00
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
log.Printf("Failed to get relative path %s: %s", path, err)
|
2012-11-30 00:07:29 +01:00
|
|
|
return nil
|
|
|
|
}
|
2012-12-01 11:53:48 +01:00
|
|
|
if remote == "." {
|
|
|
|
remote = ""
|
|
|
|
}
|
|
|
|
fs := &FsObject{remote: remote, path: path}
|
|
|
|
err = fs.lstat()
|
2012-11-30 00:07:29 +01:00
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
log.Printf("Failed to stat %s: %s", path, err)
|
2012-11-30 00:07:29 +01:00
|
|
|
return nil
|
|
|
|
}
|
2012-12-01 11:53:48 +01:00
|
|
|
return fs
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
// Walk the path returning a channel of FsObjects
|
2012-11-20 23:40:48 +01:00
|
|
|
//
|
|
|
|
// FIXME ignore symlinks?
|
|
|
|
// FIXME what about hardlinks / etc
|
2012-11-29 23:13:58 +01:00
|
|
|
func walk(root string) FsObjectsChan {
|
|
|
|
out := make(FsObjectsChan, *checkers)
|
|
|
|
go func() {
|
|
|
|
err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
|
2012-11-20 23:40:48 +01:00
|
|
|
if err != nil {
|
2012-11-29 23:13:58 +01:00
|
|
|
log.Printf("Failed to open directory: %s: %s", path, err)
|
|
|
|
} else {
|
2012-11-30 00:07:29 +01:00
|
|
|
if fs := NewFsObject(root, path); fs != nil {
|
|
|
|
out <- fs
|
2012-11-29 23:13:58 +01:00
|
|
|
}
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
2012-11-29 23:13:58 +01:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Failed to open directory: %s: %s", root, err)
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
2012-11-29 23:13:58 +01:00
|
|
|
close(out)
|
|
|
|
}()
|
|
|
|
return out
|
2012-11-20 23:40:48 +01:00
|
|
|
}
|
|
|
|
|
2012-11-18 18:32:31 +01:00
|
|
|
// syntaxError prints the syntax
|
|
|
|
func syntaxError() {
|
|
|
|
fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
|
|
|
|
|
|
|
|
FIXME
|
|
|
|
|
|
|
|
Full options:
|
|
|
|
`)
|
|
|
|
flag.PrintDefaults()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Exit with the message
|
|
|
|
func fatal(message string, args ...interface{}) {
|
|
|
|
syntaxError()
|
|
|
|
fmt.Fprintf(os.Stderr, message, args...)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
|
2012-11-20 23:40:48 +01:00
|
|
|
// checkArgs checks there are enough arguments and prints a message if not
|
|
|
|
func checkArgs(args []string, n int, message string) {
|
|
|
|
if len(args) != n {
|
|
|
|
syntaxError()
|
|
|
|
fmt.Fprintf(os.Stderr, "%d arguments required: %s\n", n, message)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
// Read FsObjects on in and write them to out if they need uploading
|
|
|
|
//
|
|
|
|
// FIXME potentially doing lots of MD5SUMS at once
|
|
|
|
func checker(c *swift.Connection, container string, in, out FsObjectsChan, wg *sync.WaitGroup) {
|
|
|
|
defer wg.Done()
|
|
|
|
for fs := range in {
|
|
|
|
// Check to see if can store this
|
2012-12-01 11:53:48 +01:00
|
|
|
if !fs.storable() {
|
2012-11-29 23:13:58 +01:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Check to see if changed or not
|
2012-12-01 11:53:48 +01:00
|
|
|
if fs.Equal(c, container, true) {
|
2012-11-29 23:13:58 +01:00
|
|
|
fs.Debugf("Unchanged skipping")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
out <- fs
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read FsObjects on in and upload them
|
|
|
|
func uploader(c *swift.Connection, container string, in FsObjectsChan, wg *sync.WaitGroup) {
|
|
|
|
defer wg.Done()
|
|
|
|
for fs := range in {
|
2012-11-20 23:40:48 +01:00
|
|
|
fs.put(c, container)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-29 23:13:58 +01:00
|
|
|
// Syncs a directory into a container
|
|
|
|
func upload(c *swift.Connection, root, container string) {
|
|
|
|
to_be_checked := walk(root)
|
|
|
|
to_be_uploaded := make(FsObjectsChan, *uploaders)
|
|
|
|
|
|
|
|
var checkerWg sync.WaitGroup
|
|
|
|
checkerWg.Add(*checkers)
|
|
|
|
for i := 0; i < *checkers; i++ {
|
|
|
|
go checker(c, container, to_be_checked, to_be_uploaded, &checkerWg)
|
|
|
|
}
|
|
|
|
|
|
|
|
var uploaderWg sync.WaitGroup
|
|
|
|
uploaderWg.Add(*uploaders)
|
|
|
|
for i := 0; i < *uploaders; i++ {
|
|
|
|
go uploader(c, container, to_be_uploaded, &uploaderWg)
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Printf("Waiting for checks to finish")
|
|
|
|
checkerWg.Wait()
|
|
|
|
close(to_be_uploaded)
|
|
|
|
log.Printf("Waiting for uploads to finish")
|
|
|
|
uploaderWg.Wait()
|
|
|
|
}
|
|
|
|
|
2012-11-30 00:07:29 +01:00
|
|
|
// Get an object to the filepath making directories if necessary
|
2012-12-01 11:53:48 +01:00
|
|
|
func (fs *FsObject) get(c *swift.Connection, container string) {
|
|
|
|
log.Printf("Download %s to %s", fs.remote, fs.path)
|
2012-11-30 00:07:29 +01:00
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
dir := path.Dir(fs.path)
|
2012-11-30 00:07:29 +01:00
|
|
|
err := os.MkdirAll(dir, 0770)
|
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Couldn't make directory: %s", err)
|
2012-11-30 00:07:29 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
out, err := os.Create(fs.path)
|
2012-11-30 00:07:29 +01:00
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Failed to open: %s", err)
|
2012-11-30 00:07:29 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2012-12-01 11:53:48 +01:00
|
|
|
h, getErr := c.ObjectGet(container, fs.remote, out, true, nil)
|
2012-12-01 09:18:58 +01:00
|
|
|
if getErr != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Failed to download: %s", getErr)
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
|
|
|
|
2012-12-01 09:18:58 +01:00
|
|
|
closeErr := out.Close()
|
|
|
|
if closeErr != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Error closing: %s", closeErr)
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
|
|
|
|
2012-12-01 09:18:58 +01:00
|
|
|
if getErr != nil || closeErr != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Removing failed download")
|
|
|
|
err = os.Remove(fs.path)
|
2012-11-30 00:07:29 +01:00
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Failed to remove failed download: %s", err)
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
2012-12-01 09:18:58 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the mtime
|
|
|
|
modTime, err := h.ObjectMetadata().GetModTime()
|
|
|
|
if err != nil {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.Debugf("Failed to read mtime from object: %s", err)
|
2012-12-01 09:18:58 +01:00
|
|
|
} else {
|
2012-12-01 11:53:48 +01:00
|
|
|
fs.SetModTime(modTime)
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Syncs a container into a directory
|
|
|
|
//
|
|
|
|
// FIXME don't want to update the modification times on the
|
|
|
|
// remote server if they are different - want to modify the local
|
|
|
|
// file!
|
|
|
|
//
|
|
|
|
// FIXME need optional stat in FsObject and to be able to make FsObjects from ObjectsAll
|
|
|
|
func download(c *swift.Connection, container, root string) {
|
|
|
|
// FIXME this would be nice running into a channel!
|
|
|
|
objects, err := c.ObjectsAll(container, nil)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't read container %q: %s", container, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = os.MkdirAll(root, 0770)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't make directory %q: %s", root, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := range objects {
|
|
|
|
object := &objects[i]
|
|
|
|
filepath := path.Join(root, object.Name)
|
2012-12-01 11:53:48 +01:00
|
|
|
fs := FsObject{remote: object.Name, path: filepath}
|
|
|
|
_ = fs.lstat()
|
|
|
|
if fs.info == nil {
|
|
|
|
fs.Debugf("Couldn't find local file - download")
|
|
|
|
} else {
|
|
|
|
fs.Debugf("Found local file - checking")
|
|
|
|
if !fs.storable() {
|
|
|
|
fs.Debugf("Not overwriting different type local file")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if fs.Equal(c, container, false) {
|
|
|
|
fs.Debugf("Skip: not changed")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fs.get(c, container)
|
2012-11-30 00:07:29 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-20 23:40:48 +01:00
|
|
|
// Lists the containers
|
|
|
|
func listContainers(c *swift.Connection) {
|
2012-11-29 00:40:09 +01:00
|
|
|
containers, err := c.ContainersAll(nil)
|
2012-11-20 23:40:48 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't list containers: %s", err)
|
|
|
|
}
|
|
|
|
for _, container := range containers {
|
|
|
|
fmt.Printf("%9d %12d %s\n", container.Count, container.Bytes, container.Name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Lists files in a container
|
|
|
|
func list(c *swift.Connection, container string) {
|
2012-11-29 00:40:09 +01:00
|
|
|
//objects, err := c.ObjectsAll(container, &swift.ObjectsOpts{Prefix: "", Delimiter: '/'})
|
|
|
|
objects, err := c.ObjectsAll(container, nil)
|
2012-11-20 23:40:48 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't read container %q: %s", container, err)
|
|
|
|
}
|
|
|
|
for _, object := range objects {
|
|
|
|
if object.PseudoDirectory {
|
|
|
|
fmt.Printf("%9s %19s %s\n", "Directory", "-", object.Name)
|
|
|
|
} else {
|
|
|
|
fmt.Printf("%9d %19s %s\n", object.Bytes, object.LastModified.Format("2006-01-02 15:04:05"), object.Name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Makes a container
|
|
|
|
func mkdir(c *swift.Connection, container string) {
|
|
|
|
err := c.ContainerCreate(container, nil)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't create container %q: %s", container, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Removes a container
|
|
|
|
func rmdir(c *swift.Connection, container string) {
|
|
|
|
err := c.ContainerDelete(container)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Couldn't delete container %q: %s", container, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-18 18:32:31 +01:00
|
|
|
func main() {
|
|
|
|
flag.Usage = syntaxError
|
|
|
|
flag.Parse()
|
2012-11-20 23:40:48 +01:00
|
|
|
args := flag.Args()
|
2012-11-29 23:13:58 +01:00
|
|
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
2012-11-18 18:32:31 +01:00
|
|
|
|
|
|
|
// Setup profiling if desired
|
|
|
|
if *cpuprofile != "" {
|
|
|
|
f, err := os.Create(*cpuprofile)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
pprof.StartCPUProfile(f)
|
|
|
|
defer pprof.StopCPUProfile()
|
|
|
|
}
|
|
|
|
|
2012-11-20 23:40:48 +01:00
|
|
|
fmt.Println(args)
|
|
|
|
if len(args) < 1 {
|
|
|
|
fatal("No command supplied\n")
|
|
|
|
}
|
2012-11-18 18:32:31 +01:00
|
|
|
|
|
|
|
if *userName == "" {
|
2012-11-20 23:40:48 +01:00
|
|
|
log.Fatal("Need -user or environmental variable ST_USER")
|
2012-11-18 18:32:31 +01:00
|
|
|
}
|
|
|
|
if *apiKey == "" {
|
2012-11-20 23:40:48 +01:00
|
|
|
log.Fatal("Need -key or environmental variable ST_KEY")
|
2012-11-18 18:32:31 +01:00
|
|
|
}
|
|
|
|
if *authUrl == "" {
|
2012-11-20 23:40:48 +01:00
|
|
|
log.Fatal("Need -auth or environmental variable ST_AUTH")
|
2012-11-18 18:32:31 +01:00
|
|
|
}
|
2012-11-20 23:40:48 +01:00
|
|
|
c := &swift.Connection{
|
2012-11-18 18:32:31 +01:00
|
|
|
UserName: *userName,
|
|
|
|
ApiKey: *apiKey,
|
|
|
|
AuthUrl: *authUrl,
|
|
|
|
}
|
|
|
|
err := c.Authenticate()
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal("Failed to authenticate", err)
|
|
|
|
}
|
|
|
|
|
2012-11-20 23:40:48 +01:00
|
|
|
command := args[0]
|
|
|
|
args = args[1:]
|
|
|
|
|
|
|
|
switch command {
|
|
|
|
case "up", "upload":
|
|
|
|
checkArgs(args, 2, "Need directory to read from and container to write to")
|
|
|
|
upload(c, args[0], args[1])
|
2012-11-30 00:07:29 +01:00
|
|
|
case "down", "download":
|
|
|
|
checkArgs(args, 2, "Need container to read from and directory to write to")
|
|
|
|
download(c, args[0], args[1])
|
2012-11-20 23:40:48 +01:00
|
|
|
case "list", "ls":
|
|
|
|
if len(args) == 0 {
|
|
|
|
listContainers(c)
|
|
|
|
} else {
|
|
|
|
checkArgs(args, 1, "Need container to list")
|
|
|
|
list(c, args[0])
|
|
|
|
}
|
|
|
|
case "mkdir":
|
|
|
|
checkArgs(args, 1, "Need container to make")
|
|
|
|
mkdir(c, args[0])
|
|
|
|
case "rmdir":
|
|
|
|
checkArgs(args, 1, "Need container to delte")
|
|
|
|
rmdir(c, args[0])
|
|
|
|
default:
|
|
|
|
log.Fatalf("Unknown command %q", command)
|
|
|
|
}
|
|
|
|
|
2012-11-18 18:32:31 +01:00
|
|
|
}
|