github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-22 13:03:02 +01:00
Code Releases Activity

Rename fs.FsX to fs.X

Browse Source
This commit is contained in:
Nick Craig-Wood 2013-06-28 08:57:32 +01:00
parent 6085dc1b5a
commit b1f681b912
7 changed files with 140 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
drive/fs.go
View File

@ -289,7 +289,7 @@ func NewFsDrive(path string) (*FsDrive, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObject { func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
fs := &FsObjectDrive{ fs := &FsObjectDrive{
drive: f, drive: f,
remote: remote, remote: remote,
@ -299,7 +299,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
} else { } else {
err := fs.readMetaData() // reads info and meta, returning an error err := fs.readMetaData() // reads info and meta, returning an error
if err != nil { if err != nil {
// logged already fs.FsDebug("Failed to read info: %s", err) // logged already fs.Debug("Failed to read info: %s", err)
return nil return nil
} }
} }
@ -309,7 +309,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDrive) NewFsObject(remote string) fs.FsObject { func (f *FsDrive) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil) return f.NewFsObjectWithInfo(remote, nil)
} }
@ -319,7 +319,7 @@ func (f *FsDrive) NewFsObject(remote string) fs.FsObject {
// //
// This fetches the minimum amount of stuff but does more API calls // This fetches the minimum amount of stuff but does more API calls
// which makes it slow // which makes it slow
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsChan) error { func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
_, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool { _, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool {
@ -357,7 +357,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsCh
// //
// This is fast in terms of number of API calls, but slow in terms of // This is fast in terms of number of API calls, but slow in terms of
// fetching more data than it needs // fetching more data than it needs
func (f *FsDrive) listDirFull(dirId string, path string, out fs.FsObjectsChan) error { func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) error {
// Orphans waiting for their parent // Orphans waiting for their parent
orphans := make(map[string][]*drive.File) orphans := make(map[string][]*drive.File)
@ -547,8 +547,8 @@ func (f *FsDrive) findRoot(create bool) error {
} }
// Walk the path returning a channel of FsObjects // Walk the path returning a channel of FsObjects
func (f *FsDrive) List() fs.FsObjectsChan { func (f *FsDrive) List() fs.ObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
err := f.findRoot(false) err := f.findRoot(false)
@ -571,8 +571,8 @@ func (f *FsDrive) List() fs.FsObjectsChan {
} }
// Walk the path returning a channel of FsObjects // Walk the path returning a channel of FsObjects
func (f *FsDrive) ListDir() fs.FsDirChan { func (f *FsDrive) ListDir() fs.DirChan {
out := make(fs.FsDirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
err := f.findRoot(false) err := f.findRoot(false)
@ -581,7 +581,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
log.Printf("Couldn't find root: %s", err) log.Printf("Couldn't find root: %s", err)
} else { } else {
_, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool { _, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool {
dir := &fs.FsDir{ dir := &fs.Dir{
Name: item.Title, Name: item.Title,
Bytes: -1, Bytes: -1,
Count: -1, Count: -1,
@ -604,7 +604,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created // The new object may have been created
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary FsObject under construction
fs := &FsObjectDrive{drive: f, remote: remote} fs := &FsObjectDrive{drive: f, remote: remote}
@ -744,7 +744,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
directory, leaf := splitPath(o.remote) directory, leaf := splitPath(o.remote)
directoryId, err := o.drive.findDir(directory, false) directoryId, err := o.drive.findDir(directory, false)
if err != nil { if err != nil {
fs.FsDebug(o, "Couldn't find directory: %s", err) fs.Debug(o, "Couldn't find directory: %s", err)
return fmt.Errorf("Couldn't find directory: %s", err) return fmt.Errorf("Couldn't find directory: %s", err)
} }
@ -759,7 +759,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
return err return err
} }
if !found { if !found {
fs.FsDebug(o, "Couldn't find object") fs.Debug(o, "Couldn't find object")
return fmt.Errorf("Couldn't find object") return fmt.Errorf("Couldn't find object")
} }
return nil return nil
@ -773,12 +773,12 @@ func (o *FsObjectDrive) readMetaData() (err error) {
func (o *FsObjectDrive) ModTime() time.Time { func (o *FsObjectDrive) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.FsLog(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
return time.Now() return time.Now()
} }
modTime, err := time.Parse(time.RFC3339, o.modifiedDate) modTime, err := time.Parse(time.RFC3339, o.modifiedDate)
if err != nil { if err != nil {
fs.FsLog(o, "Failed to read mtime from object: %s", err) fs.Log(o, "Failed to read mtime from object: %s", err)
return time.Now() return time.Now()
} }
return modTime return modTime
@ -789,7 +789,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
return return
} }
// New metadata // New metadata
@ -800,7 +800,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do() _, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err) fs.Log(o, "Failed to update remote mtime: %s", err)
} }
} }
@ -832,4 +832,4 @@ func (o *FsObjectDrive) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsDrive{} var _ fs.Fs = &FsDrive{}
var _ fs.Purger = &FsDrive{} var _ fs.Purger = &FsDrive{}
var _ fs.FsObject = &FsObjectDrive{} var _ fs.Object = &FsObjectDrive{}

16
fs/accounting.go
View File

@ -121,32 +121,32 @@ func (s *StatsInfo) Error() {
} }
// Checking adds a check into the stats // Checking adds a check into the stats
func (s *StatsInfo) Checking(fs FsObject) { func (s *StatsInfo) Checking(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
s.checking[fs.Remote()] = true s.checking[o.Remote()] = true
} }
// DoneChecking removes a check from the stats // DoneChecking removes a check from the stats
func (s *StatsInfo) DoneChecking(fs FsObject) { func (s *StatsInfo) DoneChecking(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
delete(s.checking, fs.Remote()) delete(s.checking, o.Remote())
s.checks += 1 s.checks += 1
} }
// Transferring adds a transfer into the stats // Transferring adds a transfer into the stats
func (s *StatsInfo) Transferring(fs FsObject) { func (s *StatsInfo) Transferring(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
s.transferring[fs.Remote()] = true s.transferring[o.Remote()] = true
} }
// DoneTransferring removes a transfer from the stats // DoneTransferring removes a transfer from the stats
func (s *StatsInfo) DoneTransferring(fs FsObject) { func (s *StatsInfo) DoneTransferring(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
delete(s.transferring, fs.Remote()) delete(s.transferring, o.Remote())
s.transfers += 1 s.transfers += 1
} }

86
fs/fs.go
View File

@ -13,24 +13,24 @@ import (
// Globals // Globals
var ( var (
// Global config // Global config
Config = &FsConfig{} Config = &ConfigInfo{}
// Filesystem registry // Filesystem registry
fsRegistry []fsRegistryItem fsRegistry []registryItem
) )
// Filesystem config options // Filesystem config options
type FsConfig struct { type ConfigInfo struct {
Verbose bool Verbose bool
Quiet bool Quiet bool
ModifyWindow time.Duration ModifyWindow time.Duration
Checkers int Checkers int
Transfers int Transfers int
} }
// FIXME need local to go last // FIXME need local to go last
// Filesystem registry item // Filesystem registry item
type fsRegistryItem struct { type registryItem struct {
match *regexp.Regexp // if this matches then can call newFs match *regexp.Regexp // if this matches then can call newFs
newFs func(string) (Fs, error) // create a new file system newFs func(string) (Fs, error) // create a new file system
} }
@ -41,7 +41,7 @@ type fsRegistryItem struct {
// //
// Fs modules should use this in an init() function // Fs modules should use this in an init() function
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) { func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
fsRegistry = append(fsRegistry, fsRegistryItem{match: match, newFs: newFs}) fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
} }
// A Filesystem, describes the local filesystem and the remote object store // A Filesystem, describes the local filesystem and the remote object store
@ -50,20 +50,20 @@ type Fs interface {
String() string String() string
// List the Fs into a channel // List the Fs into a channel
List() FsObjectsChan List() ObjectsChan
// List the Fs directories/buckets/containers into a channel // List the Fs directories/buckets/containers into a channel
ListDir() FsDirChan ListDir() DirChan
// Find the FsObject at remote. Returns nil if can't be found // Find the Object at remote. Returns nil if can't be found
NewFsObject(remote string) FsObject NewFsObject(remote string) Object
// Put in to the remote path with the modTime given of the given size // Put in to the remote path with the modTime given of the given size
// //
// May create the object even if it returns an error - if so // May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return // will return the object and the error, otherwise will return
// nil and the error // nil and the error
Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
// Make the directory (container, bucket) // Make the directory (container, bucket)
Mkdir() error Mkdir() error
@ -79,7 +79,7 @@ type Fs interface {
// A filesystem like object which can either be a remote object or a // A filesystem like object which can either be a remote object or a
// local file/directory // local file/directory
type FsObject interface { type Object interface {
// Remote returns the remote path // Remote returns the remote path
Remote() string Remote() string
@ -114,22 +114,22 @@ type Purger interface {
Purge() error Purge() error
} }
// A channel of FsObjects // A channel of Objects
type FsObjectsChan chan FsObject type ObjectsChan chan Object
// A slice of FsObjects // A slice of Objects
type FsObjects []FsObject type Objects []Object
// A structure of directory/container/bucket lists // A structure of directory/container/bucket lists
type FsDir struct { type Dir struct {
Name string // name of the directory Name string // name of the directory
When time.Time // modification or creation time - IsZero for unknown When time.Time // modification or creation time - IsZero for unknown
Bytes int64 // size of directory and contents -1 for unknown Bytes int64 // size of directory and contents -1 for unknown
Count int64 // number of objects -1 for unknown Count int64 // number of objects -1 for unknown
} }
// A channel of FsDir objects // A channel of Dir objects
type FsDirChan chan *FsDir type DirChan chan *Dir
// NewFs makes a new Fs object from the path // NewFs makes a new Fs object from the path
// //
@ -143,16 +143,16 @@ func NewFs(path string) (Fs, error) {
panic("Not found") // FIXME panic("Not found") // FIXME
} }
// Write debuging output for this FsObject // Write debuging output for this Object
func FsDebug(fs FsObject, text string, args ...interface{}) { func Debug(fs Object, text string, args ...interface{}) {
if Config.Verbose { if Config.Verbose {
out := fmt.Sprintf(text, args...) out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out) log.Printf("%s: %s", fs.Remote(), out)
} }
} }
// Write log output for this FsObject // Write log output for this Object
func FsLog(fs FsObject, text string, args ...interface{}) { func Log(fs Object, text string, args ...interface{}) {
if !Config.Quiet { if !Config.Quiet {
out := fmt.Sprintf(text, args...) out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out) log.Printf("%s: %s", fs.Remote(), out)
@ -173,21 +173,21 @@ func checkClose(c io.Closer, err *error) {
// May return an error which will already have been logged // May return an error which will already have been logged
// //
// If an error is returned it will return false // If an error is returned it will return false
func CheckMd5sums(src, dst FsObject) (bool, error) { func CheckMd5sums(src, dst Object) (bool, error) {
srcMd5, err := src.Md5sum() srcMd5, err := src.Md5sum()
if err != nil { if err != nil {
Stats.Error() Stats.Error()
FsLog(src, "Failed to calculate src md5: %s", err) Log(src, "Failed to calculate src md5: %s", err)
return false, err return false, err
} }
dstMd5, err := dst.Md5sum() dstMd5, err := dst.Md5sum()
if err != nil { if err != nil {
Stats.Error() Stats.Error()
FsLog(dst, "Failed to calculate dst md5: %s", err) Log(dst, "Failed to calculate dst md5: %s", err)
return false, err return false, err
} }
// FsDebug("Src MD5 %s", srcMd5) // Debug("Src MD5 %s", srcMd5)
// FsDebug("Dst MD5 %s", obj.Hash) // Debug("Dst MD5 %s", obj.Hash)
return srcMd5 == dstMd5, nil return srcMd5 == dstMd5, nil
} }
@ -207,9 +207,9 @@ func CheckMd5sums(src, dst FsObject) (bool, error) {
// //
// Otherwise the file is considered to be not equal including if there // Otherwise the file is considered to be not equal including if there
// were errors reading info. // were errors reading info.
func Equal(src, dst FsObject) bool { func Equal(src, dst Object) bool {
if src.Size() != dst.Size() { if src.Size() != dst.Size() {
FsDebug(src, "Sizes differ") Debug(src, "Sizes differ")
return false return false
} }
@ -219,9 +219,9 @@ func Equal(src, dst FsObject) bool {
dt := dstModTime.Sub(srcModTime) dt := dstModTime.Sub(srcModTime)
ModifyWindow := Config.ModifyWindow ModifyWindow := Config.ModifyWindow
if dt >= ModifyWindow || dt <= -ModifyWindow { if dt >= ModifyWindow || dt <= -ModifyWindow {
FsDebug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime) Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
} else { } else {
FsDebug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow) Debug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
return true return true
} }
@ -229,7 +229,7 @@ func Equal(src, dst FsObject) bool {
// check the MD5SUM // check the MD5SUM
same, _ := CheckMd5sums(src, dst) same, _ := CheckMd5sums(src, dst)
if !same { if !same {
FsDebug(src, "Md5sums differ") Debug(src, "Md5sums differ")
return false return false
} }
@ -237,16 +237,16 @@ func Equal(src, dst FsObject) bool {
// mtime of the dst object here // mtime of the dst object here
dst.SetModTime(srcModTime) dst.SetModTime(srcModTime)
FsDebug(src, "Size and MD5SUM of src and dst objects identical") Debug(src, "Size and MD5SUM of src and dst objects identical")
return true return true
} }
// Copy src object to f // Copy src object to f
func Copy(f Fs, src FsObject) { func Copy(f Fs, src Object) {
in0, err := src.Open() in0, err := src.Open()
if err != nil { if err != nil {
Stats.Error() Stats.Error()
FsLog(src, "Failed to open: %s", err) Log(src, "Failed to open: %s", err)
return return
} }
in := NewAccount(in0) // account the transfer in := NewAccount(in0) // account the transfer
@ -258,16 +258,16 @@ func Copy(f Fs, src FsObject) {
} }
if err != nil { if err != nil {
Stats.Error() Stats.Error()
FsLog(src, "Failed to copy: %s", err) Log(src, "Failed to copy: %s", err)
if dst != nil { if dst != nil {
FsDebug(dst, "Removing failed copy") Debug(dst, "Removing failed copy")
removeErr := dst.Remove() removeErr := dst.Remove()
if removeErr != nil { if removeErr != nil {
Stats.Error() Stats.Error()
FsLog(dst, "Failed to remove failed copy: %s", removeErr) Log(dst, "Failed to remove failed copy: %s", removeErr)
} }
} }
return return
} }
FsDebug(src, "Copied") Debug(src, "Copied")
} }

30
local/fs.go
View File

@ -55,7 +55,7 @@ func (f *FsLocal) String() string {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObject { func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
path := filepath.Join(f.root, remote) path := filepath.Join(f.root, remote)
o := &FsObjectLocal{remote: remote, path: path} o := &FsObjectLocal{remote: remote, path: path}
if info != nil { if info != nil {
@ -63,7 +63,7 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
} else { } else {
err := o.lstat() err := o.lstat()
if err != nil { if err != nil {
fs.FsDebug(o, "Failed to stat %s: %s", path, err) fs.Debug(o, "Failed to stat %s: %s", path, err)
return nil return nil
} }
} }
@ -73,15 +73,15 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.FsObject { func (f *FsLocal) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil) return f.NewFsObjectWithInfo(remote, nil)
} }
// List the path returning a channel of FsObjects // List the path returning a channel of FsObjects
// //
// Ignores everything which isn't Storable, eg links etc // Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.FsObjectsChan { func (f *FsLocal) List() fs.ObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error { err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
if err != nil { if err != nil {
@ -116,8 +116,8 @@ func (f *FsLocal) List() fs.FsObjectsChan {
} }
// Walk the path returning a channel of FsObjects // Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.FsDirChan { func (f *FsLocal) ListDir() fs.DirChan {
out := make(fs.FsDirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
items, err := ioutil.ReadDir(f.root) items, err := ioutil.ReadDir(f.root)
@ -127,7 +127,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
} else { } else {
for _, item := range items { for _, item := range items {
if item.IsDir() { if item.IsDir() {
dir := &fs.FsDir{ dir := &fs.Dir{
Name: item.Name(), Name: item.Name(),
When: item.ModTime(), When: item.ModTime(),
Bytes: 0, Bytes: 0,
@ -159,7 +159,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
} }
// Puts the FsObject to the local filesystem // Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
dstPath := filepath.Join(f.root, remote) dstPath := filepath.Join(f.root, remote)
// Temporary FsObject under construction // Temporary FsObject under construction
fs := &FsObjectLocal{remote: remote, path: dstPath} fs := &FsObjectLocal{remote: remote, path: dstPath}
@ -270,7 +270,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
in, err := os.Open(o.path) in, err := os.Open(o.path)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to open: %s", err) fs.Log(o, "Failed to open: %s", err)
return "", err return "", err
} }
defer in.Close() // FIXME ignoring error defer in.Close() // FIXME ignoring error
@ -278,7 +278,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
_, err = io.Copy(hash, in) _, err = io.Copy(hash, in)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to read: %s", err) fs.Log(o, "Failed to read: %s", err)
return "", err return "", err
} }
return fmt.Sprintf("%x", hash.Sum(nil)), nil return fmt.Sprintf("%x", hash.Sum(nil)), nil
@ -298,7 +298,7 @@ func (o *FsObjectLocal) ModTime() time.Time {
func (o *FsObjectLocal) SetModTime(modTime time.Time) { func (o *FsObjectLocal) SetModTime(modTime time.Time) {
err := os.Chtimes(o.path, modTime, modTime) err := os.Chtimes(o.path, modTime, modTime)
if err != nil { if err != nil {
fs.FsDebug(o, "Failed to set mtime on file: %s", err) fs.Debug(o, "Failed to set mtime on file: %s", err)
} }
} }
@ -306,10 +306,10 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
func (o *FsObjectLocal) Storable() bool { func (o *FsObjectLocal) Storable() bool {
mode := o.info.Mode() mode := o.info.Mode()
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
fs.FsDebug(o, "Can't transfer non file/directory") fs.Debug(o, "Can't transfer non file/directory")
return false return false
} else if mode&os.ModeDir != 0 { } else if mode&os.ModeDir != 0 {
fs.FsDebug(o, "FIXME Skipping directory") fs.Debug(o, "FIXME Skipping directory")
return false return false
} }
return true return true
@ -335,4 +335,4 @@ func (o *FsObjectLocal) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{} var _ fs.Fs = &FsLocal{}
var _ fs.FsObject = &FsObjectLocal{} var _ fs.Object = &FsObjectLocal{}

48
rclone.go
View File

@ -34,17 +34,17 @@ var (
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same") modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
) )
// A pair of fs.FsObjects // A pair of fs.Objects
type PairFsObjects struct { type PairFsObjects struct {
src, dst fs.FsObject src, dst fs.Object
} }
type PairFsObjectsChan chan PairFsObjects type PairFsObjectsChan chan PairFsObjects
// Check to see if src needs to be copied to dst and if so puts it in out // Check to see if src needs to be copied to dst and if so puts it in out
func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) { func checkOne(src, dst fs.Object, out fs.ObjectsChan) {
if dst == nil { if dst == nil {
fs.FsDebug(src, "Couldn't find local file - download") fs.Debug(src, "Couldn't find local file - download")
out <- src out <- src
return return
} }
@ -54,7 +54,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
} }
// Check to see if changed or not // Check to see if changed or not
if fs.Equal(src, dst) { if fs.Equal(src, dst) {
fs.FsDebug(src, "Unchanged skipping") fs.Debug(src, "Unchanged skipping")
return return
} }
out <- src out <- src
@ -63,7 +63,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
// Read FsObjects~s on in send to out if they need uploading // Read FsObjects~s on in send to out if they need uploading
// //
// FIXME potentially doing lots of MD5SUMS at once // FIXME potentially doing lots of MD5SUMS at once
func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup) { func PairChecker(in PairFsObjectsChan, out fs.ObjectsChan, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
for pair := range in { for pair := range in {
src := pair.src src := pair.src
@ -76,7 +76,7 @@ func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup)
// Read FsObjects~s on in send to out if they need uploading // Read FsObjects~s on in send to out if they need uploading
// //
// FIXME potentially doing lots of MD5SUMS at once // FIXME potentially doing lots of MD5SUMS at once
func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { func Checker(in, out fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
for src := range in { for src := range in {
fs.Stats.Checking(src) fs.Stats.Checking(src)
@ -87,7 +87,7 @@ func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
} }
// Read FsObjects on in and copy them // Read FsObjects on in and copy them
func Copier(in fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) { func Copier(in fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
for src := range in { for src := range in {
fs.Stats.Transferring(src) fs.Stats.Transferring(src)
@ -105,7 +105,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
} }
to_be_checked := fsrc.List() to_be_checked := fsrc.List()
to_be_uploaded := make(fs.FsObjectsChan, *transfers) to_be_uploaded := make(fs.ObjectsChan, *transfers)
var checkerWg sync.WaitGroup var checkerWg sync.WaitGroup
checkerWg.Add(*checkers) checkerWg.Add(*checkers)
@ -127,7 +127,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
} }
// Delete all the files passed in the channel // Delete all the files passed in the channel
func DeleteFiles(to_be_deleted fs.FsObjectsChan) { func DeleteFiles(to_be_deleted fs.ObjectsChan) {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(*transfers) wg.Add(*transfers)
for i := 0; i < *transfers; i++ { for i := 0; i < *transfers; i++ {
@ -135,16 +135,16 @@ func DeleteFiles(to_be_deleted fs.FsObjectsChan) {
defer wg.Done() defer wg.Done()
for dst := range to_be_deleted { for dst := range to_be_deleted {
if *dry_run { if *dry_run {
fs.FsDebug(dst, "Not deleting as -dry-run") fs.Debug(dst, "Not deleting as -dry-run")
} else { } else {
fs.Stats.Checking(dst) fs.Stats.Checking(dst)
err := dst.Remove() err := dst.Remove()
fs.Stats.DoneChecking(dst) fs.Stats.DoneChecking(dst)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(dst, "Couldn't delete: %s", err) fs.Log(dst, "Couldn't delete: %s", err)
} else { } else {
fs.FsDebug(dst, "Deleted") fs.Debug(dst, "Deleted")
} }
} }
} }
@ -167,14 +167,14 @@ func Sync(fdst, fsrc fs.Fs) {
// Read the destination files first // Read the destination files first
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
delFiles := make(map[string]fs.FsObject) delFiles := make(map[string]fs.Object)
for dst := range fdst.List() { for dst := range fdst.List() {
delFiles[dst.Remote()] = dst delFiles[dst.Remote()] = dst
} }
// Read source files checking them off against dest files // Read source files checking them off against dest files
to_be_checked := make(PairFsObjectsChan, *transfers) to_be_checked := make(PairFsObjectsChan, *transfers)
to_be_uploaded := make(fs.FsObjectsChan, *transfers) to_be_uploaded := make(fs.ObjectsChan, *transfers)
var checkerWg sync.WaitGroup var checkerWg sync.WaitGroup
checkerWg.Add(*checkers) checkerWg.Add(*checkers)
@ -215,7 +215,7 @@ func Sync(fdst, fsrc fs.Fs) {
} }
// Delete the spare files // Delete the spare files
toDelete := make(fs.FsObjectsChan, *transfers) toDelete := make(fs.ObjectsChan, *transfers)
go func() { go func() {
for _, fs := range delFiles { for _, fs := range delFiles {
toDelete <- fs toDelete <- fs
@ -231,19 +231,19 @@ func Check(fdst, fsrc fs.Fs) {
// Read the destination files first // Read the destination files first
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
dstFiles := make(map[string]fs.FsObject) dstFiles := make(map[string]fs.Object)
for dst := range fdst.List() { for dst := range fdst.List() {
dstFiles[dst.Remote()] = dst dstFiles[dst.Remote()] = dst
} }
// Read the source files checking them against dstFiles // Read the source files checking them against dstFiles
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
srcFiles := make(map[string]fs.FsObject) srcFiles := make(map[string]fs.Object)
commonFiles := make(map[string][]fs.FsObject) commonFiles := make(map[string][]fs.Object)
for src := range fsrc.List() { for src := range fsrc.List() {
remote := src.Remote() remote := src.Remote()
if dst, ok := dstFiles[remote]; ok { if dst, ok := dstFiles[remote]; ok {
commonFiles[remote] = []fs.FsObject{dst, src} commonFiles[remote] = []fs.Object{dst, src}
delete(dstFiles, remote) delete(dstFiles, remote)
} else { } else {
srcFiles[remote] = src srcFiles[remote] = src
@ -262,7 +262,7 @@ func Check(fdst, fsrc fs.Fs) {
log.Printf(remote) log.Printf(remote)
} }
checks := make(chan []fs.FsObject, *transfers) checks := make(chan []fs.Object, *transfers)
go func() { go func() {
for _, check := range commonFiles { for _, check := range commonFiles {
checks <- check checks <- check
@ -281,7 +281,7 @@ func Check(fdst, fsrc fs.Fs) {
if src.Size() != dst.Size() { if src.Size() != dst.Size() {
fs.Stats.DoneChecking(src) fs.Stats.DoneChecking(src)
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(src, "Sizes differ") fs.Log(src, "Sizes differ")
continue continue
} }
same, err := fs.CheckMd5sums(src, dst) same, err := fs.CheckMd5sums(src, dst)
@ -291,9 +291,9 @@ func Check(fdst, fsrc fs.Fs) {
} }
if !same { if !same {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(src, "Md5sums differ") fs.Log(src, "Md5sums differ")
} }
fs.FsDebug(src, "OK") fs.Debug(src, "OK")
} }
}() }()
} }

36
s3/fs.go
View File

@ -143,7 +143,7 @@ func NewFs(path string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject { func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
o := &FsObjectS3{ o := &FsObjectS3{
s3: f, s3: f,
remote: remote, remote: remote,
@ -153,7 +153,7 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
var err error var err error
o.lastModified, err = time.Parse(time.RFC3339, info.LastModified) o.lastModified, err = time.Parse(time.RFC3339, info.LastModified)
if err != nil { if err != nil {
fs.FsLog(o, "Failed to read last modified: %s", err) fs.Log(o, "Failed to read last modified: %s", err)
o.lastModified = time.Now() o.lastModified = time.Now()
} }
o.etag = info.ETag o.etag = info.ETag
@ -171,13 +171,13 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsS3) NewFsObject(remote string) fs.FsObject { func (f *FsS3) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil) return f.NewFsObjectWithInfo(remote, nil)
} }
// Walk the path returning a channel of FsObjects // Walk the path returning a channel of FsObjects
func (f *FsS3) List() fs.FsObjectsChan { func (f *FsS3) List() fs.ObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
// FIXME need to implement ALL loop // FIXME need to implement ALL loop
objects, err := f.b.List("", "", "", 10000) objects, err := f.b.List("", "", "", 10000)
@ -198,8 +198,8 @@ func (f *FsS3) List() fs.FsObjectsChan {
} }
// Lists the buckets // Lists the buckets
func (f *FsS3) ListDir() fs.FsDirChan { func (f *FsS3) ListDir() fs.DirChan {
out := make(fs.FsDirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
buckets, err := f.c.ListBuckets() buckets, err := f.c.ListBuckets()
@ -208,7 +208,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
log.Printf("Couldn't list buckets: %s", err) log.Printf("Couldn't list buckets: %s", err)
} else { } else {
for _, bucket := range buckets { for _, bucket := range buckets {
out <- &fs.FsDir{ out <- &fs.Dir{
Name: bucket.Name, Name: bucket.Name,
When: bucket.CreationDate, When: bucket.CreationDate,
Bytes: -1, Bytes: -1,
@ -221,7 +221,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
} }
// Put the FsObject into the bucket // Put the FsObject into the bucket
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary FsObject under construction
fs := &FsObjectS3{s3: f, remote: remote} fs := &FsObjectS3{s3: f, remote: remote}
@ -290,19 +290,19 @@ func (o *FsObjectS3) readMetaData() (err error) {
headers, err := o.s3.b.Head(o.remote, nil) headers, err := o.s3.b.Head(o.remote, nil)
if err != nil { if err != nil {
fs.FsDebug(o, "Failed to read info: %s", err) fs.Debug(o, "Failed to read info: %s", err)
return err return err
} }
size, err := strconv.ParseInt(headers["Content-Length"], 10, 64) size, err := strconv.ParseInt(headers["Content-Length"], 10, 64)
if err != nil { if err != nil {
fs.FsDebug(o, "Failed to read size from: %q", headers) fs.Debug(o, "Failed to read size from: %q", headers)
return err return err
} }
o.etag = headers["Etag"] o.etag = headers["Etag"]
o.bytes = size o.bytes = size
o.meta = headers o.meta = headers
if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil { if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil {
fs.FsLog(o, "Failed to read last modified from HEAD: %s", err) fs.Log(o, "Failed to read last modified from HEAD: %s", err)
o.lastModified = time.Now() o.lastModified = time.Now()
} }
return nil return nil
@ -315,18 +315,18 @@ func (o *FsObjectS3) readMetaData() (err error) {
func (o *FsObjectS3) ModTime() time.Time { func (o *FsObjectS3) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.FsLog(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
return time.Now() return time.Now()
} }
// read mtime out of metadata if available // read mtime out of metadata if available
d, ok := o.meta[metaMtime] d, ok := o.meta[metaMtime]
if !ok { if !ok {
// fs.FsDebug(o, "No metadata") // fs.Debug(o, "No metadata")
return o.lastModified return o.lastModified
} }
modTime, err := swift.FloatStringToTime(d) modTime, err := swift.FloatStringToTime(d)
if err != nil { if err != nil {
fs.FsLog(o, "Failed to read mtime from object: %s", err) fs.Log(o, "Failed to read mtime from object: %s", err)
return o.lastModified return o.lastModified
} }
return modTime return modTime
@ -337,14 +337,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
return return
} }
o.meta[metaMtime] = swift.TimeToFloatString(modTime) o.meta[metaMtime] = swift.TimeToFloatString(modTime)
_, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta) _, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta)
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err) fs.Log(o, "Failed to update remote mtime: %s", err)
} }
} }
@ -366,4 +366,4 @@ func (o *FsObjectS3) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsS3{} var _ fs.Fs = &FsS3{}
var _ fs.FsObject = &FsObjectS3{} var _ fs.Object = &FsObjectS3{}

28
swift/fs.go
View File

@ -114,7 +114,7 @@ func NewFs(path string) (fs.Fs, error) {
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsObject { func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
fs := &FsObjectSwift{ fs := &FsObjectSwift{
swift: f, swift: f,
remote: remote, remote: remote,
@ -135,13 +135,13 @@ func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsOb
// Return an FsObject from a path // Return an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.FsObject { func (f *FsSwift) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil) return f.NewFsObjectWithInfo(remote, nil)
} }
// Walk the path returning a channel of FsObjects // Walk the path returning a channel of FsObjects
func (f *FsSwift) List() fs.FsObjectsChan { func (f *FsSwift) List() fs.ObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
// FIXME use a smaller limit? // FIXME use a smaller limit?
err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) { err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
@ -166,8 +166,8 @@ func (f *FsSwift) List() fs.FsObjectsChan {
} }
// Lists the containers // Lists the containers
func (f *FsSwift) ListDir() fs.FsDirChan { func (f *FsSwift) ListDir() fs.DirChan {
out := make(fs.FsDirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
defer close(out) defer close(out)
containers, err := f.c.ContainersAll(nil) containers, err := f.c.ContainersAll(nil)
@ -176,7 +176,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
log.Printf("Couldn't list containers: %s", err) log.Printf("Couldn't list containers: %s", err)
} else { } else {
for _, container := range containers { for _, container := range containers {
out <- &fs.FsDir{ out <- &fs.Dir{
Name: container.Name, Name: container.Name,
Bytes: container.Bytes, Bytes: container.Bytes,
Count: container.Count, Count: container.Count,
@ -192,7 +192,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
// Copy the reader in to the new object which is returned // Copy the reader in to the new object which is returned
// //
// The new object may have been created // The new object may have been created
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) { func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction // Temporary FsObject under construction
fs := &FsObjectSwift{swift: f, remote: remote} fs := &FsObjectSwift{swift: f, remote: remote}
@ -246,7 +246,7 @@ func (o *FsObjectSwift) readMetaData() (err error) {
} }
info, h, err := o.swift.c.Object(o.swift.container, o.remote) info, h, err := o.swift.c.Object(o.swift.container, o.remote)
if err != nil { if err != nil {
fs.FsDebug(o, "Failed to read info: %s", err) fs.Debug(o, "Failed to read info: %s", err)
return err return err
} }
meta := h.ObjectMetadata() meta := h.ObjectMetadata()
@ -263,12 +263,12 @@ func (o *FsObjectSwift) readMetaData() (err error) {
func (o *FsObjectSwift) ModTime() time.Time { func (o *FsObjectSwift) ModTime() time.Time {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
// fs.FsLog(o, "Failed to read metadata: %s", err) // fs.Log(o, "Failed to read metadata: %s", err)
return o.info.LastModified return o.info.LastModified
} }
modTime, err := o.meta.GetModTime() modTime, err := o.meta.GetModTime()
if err != nil { if err != nil {
// fs.FsLog(o, "Failed to read mtime from object: %s", err) // fs.Log(o, "Failed to read mtime from object: %s", err)
return o.info.LastModified return o.info.LastModified
} }
return modTime return modTime
@ -279,14 +279,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err) fs.Log(o, "Failed to read metadata: %s", err)
return return
} }
o.meta.SetModTime(modTime) o.meta.SetModTime(modTime)
err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders()) err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders())
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err) fs.Log(o, "Failed to update remote mtime: %s", err)
} }
} }
@ -308,4 +308,4 @@ func (o *FsObjectSwift) Remove() error {
// Check the interfaces are satisfied // Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{} var _ fs.Fs = &FsSwift{}
var _ fs.FsObject = &FsObjectSwift{} var _ fs.Object = &FsObjectSwift{}