github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-22 13:03:02 +01:00
Code Releases Activity

Rename fs.FsX to fs.X

Browse Source
This commit is contained in:
Nick Craig-Wood 2013-06-28 08:57:32 +01:00
parent 6085dc1b5a
commit b1f681b912
7 changed files with 140 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
drive/fs.go
View File

@ -289,7 +289,7 @@ func NewFsDrive(path string) (*FsDrive, error) {
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObject {
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
fs := &FsObjectDrive{
drive: f,
remote: remote,
@ -299,7 +299,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
} else {
err := fs.readMetaData() // reads info and meta, returning an error
if err != nil {
// logged already fs.FsDebug("Failed to read info: %s", err)
// logged already fs.Debug("Failed to read info: %s", err)
return nil
}
}
@ -309,7 +309,7 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.FsObje
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsDrive) NewFsObject(remote string) fs.FsObject {
func (f *FsDrive) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil)
}
@ -319,7 +319,7 @@ func (f *FsDrive) NewFsObject(remote string) fs.FsObject {
//
// This fetches the minimum amount of stuff but does more API calls
// which makes it slow
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsChan) error {
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
var subError error
// Make the API request
_, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool {
@ -357,7 +357,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.FsObjectsCh
//
// This is fast in terms of number of API calls, but slow in terms of
// fetching more data than it needs
func (f *FsDrive) listDirFull(dirId string, path string, out fs.FsObjectsChan) error {
func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) error {
// Orphans waiting for their parent
orphans := make(map[string][]*drive.File)
@ -547,8 +547,8 @@ func (f *FsDrive) findRoot(create bool) error {
}
// Walk the path returning a channel of FsObjects
func (f *FsDrive) List() fs.FsObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers)
func (f *FsDrive) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() {
defer close(out)
err := f.findRoot(false)
@ -571,8 +571,8 @@ func (f *FsDrive) List() fs.FsObjectsChan {
}
// Walk the path returning a channel of FsObjects
func (f *FsDrive) ListDir() fs.FsDirChan {
out := make(fs.FsDirChan, fs.Config.Checkers)
func (f *FsDrive) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers)
go func() {
defer close(out)
err := f.findRoot(false)
@ -581,7 +581,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
log.Printf("Couldn't find root: %s", err)
} else {
_, err := f.listAll(f.rootId, "", true, false, func(item *drive.File) bool {
dir := &fs.FsDir{
dir := &fs.Dir{
Name: item.Title,
Bytes: -1,
Count: -1,
@ -604,7 +604,7 @@ func (f *FsDrive) ListDir() fs.FsDirChan {
// Copy the reader in to the new object which is returned
//
// The new object may have been created
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectDrive{drive: f, remote: remote}
@ -744,7 +744,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
directory, leaf := splitPath(o.remote)
directoryId, err := o.drive.findDir(directory, false)
if err != nil {
fs.FsDebug(o, "Couldn't find directory: %s", err)
fs.Debug(o, "Couldn't find directory: %s", err)
return fmt.Errorf("Couldn't find directory: %s", err)
}
@ -759,7 +759,7 @@ func (o *FsObjectDrive) readMetaData() (err error) {
return err
}
if !found {
fs.FsDebug(o, "Couldn't find object")
fs.Debug(o, "Couldn't find object")
return fmt.Errorf("Couldn't find object")
}
return nil
@ -773,12 +773,12 @@ func (o *FsObjectDrive) readMetaData() (err error) {
func (o *FsObjectDrive) ModTime() time.Time {
err := o.readMetaData()
if err != nil {
fs.FsLog(o, "Failed to read metadata: %s", err)
fs.Log(o, "Failed to read metadata: %s", err)
return time.Now()
}
modTime, err := time.Parse(time.RFC3339, o.modifiedDate)
if err != nil {
fs.FsLog(o, "Failed to read mtime from object: %s", err)
fs.Log(o, "Failed to read mtime from object: %s", err)
return time.Now()
}
return modTime
@ -789,7 +789,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
err := o.readMetaData()
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err)
fs.Log(o, "Failed to read metadata: %s", err)
return
}
// New metadata
@ -800,7 +800,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
_, err = o.drive.svc.Files.Update(o.id, info).SetModifiedDate(true).Do()
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err)
fs.Log(o, "Failed to update remote mtime: %s", err)
}
}
@ -832,4 +832,4 @@ func (o *FsObjectDrive) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsDrive{}
var _ fs.Purger = &FsDrive{}
var _ fs.FsObject = &FsObjectDrive{}
var _ fs.Object = &FsObjectDrive{}

16
fs/accounting.go
View File

@ -121,32 +121,32 @@ func (s *StatsInfo) Error() {
}
// Checking adds a check into the stats
func (s *StatsInfo) Checking(fs FsObject) {
func (s *StatsInfo) Checking(o Object) {
s.lock.Lock()
defer s.lock.Unlock()
s.checking[fs.Remote()] = true
s.checking[o.Remote()] = true
}
// DoneChecking removes a check from the stats
func (s *StatsInfo) DoneChecking(fs FsObject) {
func (s *StatsInfo) DoneChecking(o Object) {
s.lock.Lock()
defer s.lock.Unlock()
delete(s.checking, fs.Remote())
delete(s.checking, o.Remote())
s.checks += 1
}
// Transferring adds a transfer into the stats
func (s *StatsInfo) Transferring(fs FsObject) {
func (s *StatsInfo) Transferring(o Object) {
s.lock.Lock()
defer s.lock.Unlock()
s.transferring[fs.Remote()] = true
s.transferring[o.Remote()] = true
}
// DoneTransferring removes a transfer from the stats
func (s *StatsInfo) DoneTransferring(fs FsObject) {
func (s *StatsInfo) DoneTransferring(o Object) {
s.lock.Lock()
defer s.lock.Unlock()
delete(s.transferring, fs.Remote())
delete(s.transferring, o.Remote())
s.transfers += 1
}

86
fs/fs.go
View File

@ -13,24 +13,24 @@ import (
// Globals
var (
// Global config
Config = &FsConfig{}
Config = &ConfigInfo{}
// Filesystem registry
fsRegistry []fsRegistryItem
fsRegistry []registryItem
)
// Filesystem config options
type FsConfig struct {
Verbose bool
Quiet bool
type ConfigInfo struct {
Verbose bool
Quiet bool
ModifyWindow time.Duration
Checkers int
Transfers int
Checkers int
Transfers int
}
// FIXME need local to go last
// Filesystem registry item
type fsRegistryItem struct {
type registryItem struct {
match *regexp.Regexp // if this matches then can call newFs
newFs func(string) (Fs, error) // create a new file system
}
@ -41,7 +41,7 @@ type fsRegistryItem struct {
//
// Fs modules should use this in an init() function
func Register(match *regexp.Regexp, newFs func(string) (Fs, error)) {
fsRegistry = append(fsRegistry, fsRegistryItem{match: match, newFs: newFs})
fsRegistry = append(fsRegistry, registryItem{match: match, newFs: newFs})
}
// A Filesystem, describes the local filesystem and the remote object store
@ -50,20 +50,20 @@ type Fs interface {
String() string
// List the Fs into a channel
List() FsObjectsChan
List() ObjectsChan
// List the Fs directories/buckets/containers into a channel
ListDir() FsDirChan
ListDir() DirChan
// Find the FsObject at remote. Returns nil if can't be found
NewFsObject(remote string) FsObject
// Find the Object at remote. Returns nil if can't be found
NewFsObject(remote string) Object
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
Put(in io.Reader, remote string, modTime time.Time, size int64) (FsObject, error)
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
// Make the directory (container, bucket)
Mkdir() error
@ -79,7 +79,7 @@ type Fs interface {
// A filesystem like object which can either be a remote object or a
// local file/directory
type FsObject interface {
type Object interface {
// Remote returns the remote path
Remote() string
@ -114,22 +114,22 @@ type Purger interface {
Purge() error
}
// A channel of FsObjects
type FsObjectsChan chan FsObject
// A channel of Objects
type ObjectsChan chan Object
// A slice of FsObjects
type FsObjects []FsObject
// A slice of Objects
type Objects []Object
// A structure of directory/container/bucket lists
type FsDir struct {
type Dir struct {
Name string // name of the directory
When time.Time // modification or creation time - IsZero for unknown
Bytes int64 // size of directory and contents -1 for unknown
Count int64 // number of objects -1 for unknown
}
// A channel of FsDir objects
type FsDirChan chan *FsDir
// A channel of Dir objects
type DirChan chan *Dir
// NewFs makes a new Fs object from the path
//
@ -143,16 +143,16 @@ func NewFs(path string) (Fs, error) {
panic("Not found") // FIXME
}
// Write debuging output for this FsObject
func FsDebug(fs FsObject, text string, args ...interface{}) {
// Write debuging output for this Object
func Debug(fs Object, text string, args ...interface{}) {
if Config.Verbose {
out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out)
}
}
// Write log output for this FsObject
func FsLog(fs FsObject, text string, args ...interface{}) {
// Write log output for this Object
func Log(fs Object, text string, args ...interface{}) {
if !Config.Quiet {
out := fmt.Sprintf(text, args...)
log.Printf("%s: %s", fs.Remote(), out)
@ -173,21 +173,21 @@ func checkClose(c io.Closer, err *error) {
// May return an error which will already have been logged
//
// If an error is returned it will return false
func CheckMd5sums(src, dst FsObject) (bool, error) {
func CheckMd5sums(src, dst Object) (bool, error) {
srcMd5, err := src.Md5sum()
if err != nil {
Stats.Error()
FsLog(src, "Failed to calculate src md5: %s", err)
Log(src, "Failed to calculate src md5: %s", err)
return false, err
}
dstMd5, err := dst.Md5sum()
if err != nil {
Stats.Error()
FsLog(dst, "Failed to calculate dst md5: %s", err)
Log(dst, "Failed to calculate dst md5: %s", err)
return false, err
}
// FsDebug("Src MD5 %s", srcMd5)
// FsDebug("Dst MD5 %s", obj.Hash)
// Debug("Src MD5 %s", srcMd5)
// Debug("Dst MD5 %s", obj.Hash)
return srcMd5 == dstMd5, nil
}
@ -207,9 +207,9 @@ func CheckMd5sums(src, dst FsObject) (bool, error) {
//
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(src, dst FsObject) bool {
func Equal(src, dst Object) bool {
if src.Size() != dst.Size() {
FsDebug(src, "Sizes differ")
Debug(src, "Sizes differ")
return false
}
@ -219,9 +219,9 @@ func Equal(src, dst FsObject) bool {
dt := dstModTime.Sub(srcModTime)
ModifyWindow := Config.ModifyWindow
if dt >= ModifyWindow || dt <= -ModifyWindow {
FsDebug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
} else {
FsDebug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
Debug(src, "Size and modification time differ by %s (within %s)", dt, ModifyWindow)
return true
}
@ -229,7 +229,7 @@ func Equal(src, dst FsObject) bool {
// check the MD5SUM
same, _ := CheckMd5sums(src, dst)
if !same {
FsDebug(src, "Md5sums differ")
Debug(src, "Md5sums differ")
return false
}
@ -237,16 +237,16 @@ func Equal(src, dst FsObject) bool {
// mtime of the dst object here
dst.SetModTime(srcModTime)
FsDebug(src, "Size and MD5SUM of src and dst objects identical")
Debug(src, "Size and MD5SUM of src and dst objects identical")
return true
}
// Copy src object to f
func Copy(f Fs, src FsObject) {
func Copy(f Fs, src Object) {
in0, err := src.Open()
if err != nil {
Stats.Error()
FsLog(src, "Failed to open: %s", err)
Log(src, "Failed to open: %s", err)
return
}
in := NewAccount(in0) // account the transfer
@ -258,16 +258,16 @@ func Copy(f Fs, src FsObject) {
}
if err != nil {
Stats.Error()
FsLog(src, "Failed to copy: %s", err)
Log(src, "Failed to copy: %s", err)
if dst != nil {
FsDebug(dst, "Removing failed copy")
Debug(dst, "Removing failed copy")
removeErr := dst.Remove()
if removeErr != nil {
Stats.Error()
FsLog(dst, "Failed to remove failed copy: %s", removeErr)
Log(dst, "Failed to remove failed copy: %s", removeErr)
}
}
return
}
FsDebug(src, "Copied")
Debug(src, "Copied")
}

30
local/fs.go
View File

@ -55,7 +55,7 @@ func (f *FsLocal) String() string {
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObject {
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
path := filepath.Join(f.root, remote)
o := &FsObjectLocal{remote: remote, path: path}
if info != nil {
@ -63,7 +63,7 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
} else {
err := o.lstat()
if err != nil {
fs.FsDebug(o, "Failed to stat %s: %s", path, err)
fs.Debug(o, "Failed to stat %s: %s", path, err)
return nil
}
}
@ -73,15 +73,15 @@ func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.FsObje
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.FsObject {
func (f *FsLocal) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil)
}
// List the path returning a channel of FsObjects
//
// Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.FsObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers)
func (f *FsLocal) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() {
err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
if err != nil {
@ -116,8 +116,8 @@ func (f *FsLocal) List() fs.FsObjectsChan {
}
// Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.FsDirChan {
out := make(fs.FsDirChan, fs.Config.Checkers)
func (f *FsLocal) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers)
go func() {
defer close(out)
items, err := ioutil.ReadDir(f.root)
@ -127,7 +127,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
} else {
for _, item := range items {
if item.IsDir() {
dir := &fs.FsDir{
dir := &fs.Dir{
Name: item.Name(),
When: item.ModTime(),
Bytes: 0,
@ -159,7 +159,7 @@ func (f *FsLocal) ListDir() fs.FsDirChan {
}
// Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
dstPath := filepath.Join(f.root, remote)
// Temporary FsObject under construction
fs := &FsObjectLocal{remote: remote, path: dstPath}
@ -270,7 +270,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
in, err := os.Open(o.path)
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to open: %s", err)
fs.Log(o, "Failed to open: %s", err)
return "", err
}
defer in.Close() // FIXME ignoring error
@ -278,7 +278,7 @@ func (o *FsObjectLocal) Md5sum() (string, error) {
_, err = io.Copy(hash, in)
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to read: %s", err)
fs.Log(o, "Failed to read: %s", err)
return "", err
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
@ -298,7 +298,7 @@ func (o *FsObjectLocal) ModTime() time.Time {
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
err := os.Chtimes(o.path, modTime, modTime)
if err != nil {
fs.FsDebug(o, "Failed to set mtime on file: %s", err)
fs.Debug(o, "Failed to set mtime on file: %s", err)
}
}
@ -306,10 +306,10 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
func (o *FsObjectLocal) Storable() bool {
mode := o.info.Mode()
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
fs.FsDebug(o, "Can't transfer non file/directory")
fs.Debug(o, "Can't transfer non file/directory")
return false
} else if mode&os.ModeDir != 0 {
fs.FsDebug(o, "FIXME Skipping directory")
fs.Debug(o, "FIXME Skipping directory")
return false
}
return true
@ -335,4 +335,4 @@ func (o *FsObjectLocal) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.FsObject = &FsObjectLocal{}
var _ fs.Object = &FsObjectLocal{}

48
rclone.go
View File

@ -34,17 +34,17 @@ var (
modifyWindow = flag.Duration("modify-window", time.Nanosecond, "Max time diff to be considered the same")
)
// A pair of fs.FsObjects
// A pair of fs.Objects
type PairFsObjects struct {
src, dst fs.FsObject
src, dst fs.Object
}
type PairFsObjectsChan chan PairFsObjects
// Check to see if src needs to be copied to dst and if so puts it in out
func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
func checkOne(src, dst fs.Object, out fs.ObjectsChan) {
if dst == nil {
fs.FsDebug(src, "Couldn't find local file - download")
fs.Debug(src, "Couldn't find local file - download")
out <- src
return
}
@ -54,7 +54,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
}
// Check to see if changed or not
if fs.Equal(src, dst) {
fs.FsDebug(src, "Unchanged skipping")
fs.Debug(src, "Unchanged skipping")
return
}
out <- src
@ -63,7 +63,7 @@ func checkOne(src, dst fs.FsObject, out fs.FsObjectsChan) {
// Read FsObjects~s on in send to out if they need uploading
//
// FIXME potentially doing lots of MD5SUMS at once
func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup) {
func PairChecker(in PairFsObjectsChan, out fs.ObjectsChan, wg *sync.WaitGroup) {
defer wg.Done()
for pair := range in {
src := pair.src
@ -76,7 +76,7 @@ func PairChecker(in PairFsObjectsChan, out fs.FsObjectsChan, wg *sync.WaitGroup)
// Read FsObjects~s on in send to out if they need uploading
//
// FIXME potentially doing lots of MD5SUMS at once
func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
func Checker(in, out fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
defer wg.Done()
for src := range in {
fs.Stats.Checking(src)
@ -87,7 +87,7 @@ func Checker(in, out fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
}
// Read FsObjects on in and copy them
func Copier(in fs.FsObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
func Copier(in fs.ObjectsChan, fdst fs.Fs, wg *sync.WaitGroup) {
defer wg.Done()
for src := range in {
fs.Stats.Transferring(src)
@ -105,7 +105,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
}
to_be_checked := fsrc.List()
to_be_uploaded := make(fs.FsObjectsChan, *transfers)
to_be_uploaded := make(fs.ObjectsChan, *transfers)
var checkerWg sync.WaitGroup
checkerWg.Add(*checkers)
@ -127,7 +127,7 @@ func CopyFs(fdst, fsrc fs.Fs) {
}
// Delete all the files passed in the channel
func DeleteFiles(to_be_deleted fs.FsObjectsChan) {
func DeleteFiles(to_be_deleted fs.ObjectsChan) {
var wg sync.WaitGroup
wg.Add(*transfers)
for i := 0; i < *transfers; i++ {
@ -135,16 +135,16 @@ func DeleteFiles(to_be_deleted fs.FsObjectsChan) {
defer wg.Done()
for dst := range to_be_deleted {
if *dry_run {
fs.FsDebug(dst, "Not deleting as -dry-run")
fs.Debug(dst, "Not deleting as -dry-run")
} else {
fs.Stats.Checking(dst)
err := dst.Remove()
fs.Stats.DoneChecking(dst)
if err != nil {
fs.Stats.Error()
fs.FsLog(dst, "Couldn't delete: %s", err)
fs.Log(dst, "Couldn't delete: %s", err)
} else {
fs.FsDebug(dst, "Deleted")
fs.Debug(dst, "Deleted")
}
}
}
@ -167,14 +167,14 @@ func Sync(fdst, fsrc fs.Fs) {
// Read the destination files first
// FIXME could do this in parallel and make it use less memory
delFiles := make(map[string]fs.FsObject)
delFiles := make(map[string]fs.Object)
for dst := range fdst.List() {
delFiles[dst.Remote()] = dst
}
// Read source files checking them off against dest files
to_be_checked := make(PairFsObjectsChan, *transfers)
to_be_uploaded := make(fs.FsObjectsChan, *transfers)
to_be_uploaded := make(fs.ObjectsChan, *transfers)
var checkerWg sync.WaitGroup
checkerWg.Add(*checkers)
@ -215,7 +215,7 @@ func Sync(fdst, fsrc fs.Fs) {
}
// Delete the spare files
toDelete := make(fs.FsObjectsChan, *transfers)
toDelete := make(fs.ObjectsChan, *transfers)
go func() {
for _, fs := range delFiles {
toDelete <- fs
@ -231,19 +231,19 @@ func Check(fdst, fsrc fs.Fs) {
// Read the destination files first
// FIXME could do this in parallel and make it use less memory
dstFiles := make(map[string]fs.FsObject)
dstFiles := make(map[string]fs.Object)
for dst := range fdst.List() {
dstFiles[dst.Remote()] = dst
}
// Read the source files checking them against dstFiles
// FIXME could do this in parallel and make it use less memory
srcFiles := make(map[string]fs.FsObject)
commonFiles := make(map[string][]fs.FsObject)
srcFiles := make(map[string]fs.Object)
commonFiles := make(map[string][]fs.Object)
for src := range fsrc.List() {
remote := src.Remote()
if dst, ok := dstFiles[remote]; ok {
commonFiles[remote] = []fs.FsObject{dst, src}
commonFiles[remote] = []fs.Object{dst, src}
delete(dstFiles, remote)
} else {
srcFiles[remote] = src
@ -262,7 +262,7 @@ func Check(fdst, fsrc fs.Fs) {
log.Printf(remote)
}
checks := make(chan []fs.FsObject, *transfers)
checks := make(chan []fs.Object, *transfers)
go func() {
for _, check := range commonFiles {
checks <- check
@ -281,7 +281,7 @@ func Check(fdst, fsrc fs.Fs) {
if src.Size() != dst.Size() {
fs.Stats.DoneChecking(src)
fs.Stats.Error()
fs.FsLog(src, "Sizes differ")
fs.Log(src, "Sizes differ")
continue
}
same, err := fs.CheckMd5sums(src, dst)
@ -291,9 +291,9 @@ func Check(fdst, fsrc fs.Fs) {
}
if !same {
fs.Stats.Error()
fs.FsLog(src, "Md5sums differ")
fs.Log(src, "Md5sums differ")
}
fs.FsDebug(src, "OK")
fs.Debug(src, "OK")
}
}()
}

36
s3/fs.go
View File

@ -143,7 +143,7 @@ func NewFs(path string) (fs.Fs, error) {
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.Object {
o := &FsObjectS3{
s3: f,
remote: remote,
@ -153,7 +153,7 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
var err error
o.lastModified, err = time.Parse(time.RFC3339, info.LastModified)
if err != nil {
fs.FsLog(o, "Failed to read last modified: %s", err)
fs.Log(o, "Failed to read last modified: %s", err)
o.lastModified = time.Now()
}
o.etag = info.ETag
@ -171,13 +171,13 @@ func (f *FsS3) NewFsObjectWithInfo(remote string, info *s3.Key) fs.FsObject {
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsS3) NewFsObject(remote string) fs.FsObject {
func (f *FsS3) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil)
}
// Walk the path returning a channel of FsObjects
func (f *FsS3) List() fs.FsObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers)
func (f *FsS3) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() {
// FIXME need to implement ALL loop
objects, err := f.b.List("", "", "", 10000)
@ -198,8 +198,8 @@ func (f *FsS3) List() fs.FsObjectsChan {
}
// Lists the buckets
func (f *FsS3) ListDir() fs.FsDirChan {
out := make(fs.FsDirChan, fs.Config.Checkers)
func (f *FsS3) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers)
go func() {
defer close(out)
buckets, err := f.c.ListBuckets()
@ -208,7 +208,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
log.Printf("Couldn't list buckets: %s", err)
} else {
for _, bucket := range buckets {
out <- &fs.FsDir{
out <- &fs.Dir{
Name: bucket.Name,
When: bucket.CreationDate,
Bytes: -1,
@ -221,7 +221,7 @@ func (f *FsS3) ListDir() fs.FsDirChan {
}
// Put the FsObject into the bucket
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
func (f *FsS3) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectS3{s3: f, remote: remote}
@ -290,19 +290,19 @@ func (o *FsObjectS3) readMetaData() (err error) {
headers, err := o.s3.b.Head(o.remote, nil)
if err != nil {
fs.FsDebug(o, "Failed to read info: %s", err)
fs.Debug(o, "Failed to read info: %s", err)
return err
}
size, err := strconv.ParseInt(headers["Content-Length"], 10, 64)
if err != nil {
fs.FsDebug(o, "Failed to read size from: %q", headers)
fs.Debug(o, "Failed to read size from: %q", headers)
return err
}
o.etag = headers["Etag"]
o.bytes = size
o.meta = headers
if o.lastModified, err = time.Parse(http.TimeFormat, headers["Last-Modified"]); err != nil {
fs.FsLog(o, "Failed to read last modified from HEAD: %s", err)
fs.Log(o, "Failed to read last modified from HEAD: %s", err)
o.lastModified = time.Now()
}
return nil
@ -315,18 +315,18 @@ func (o *FsObjectS3) readMetaData() (err error) {
func (o *FsObjectS3) ModTime() time.Time {
err := o.readMetaData()
if err != nil {
fs.FsLog(o, "Failed to read metadata: %s", err)
fs.Log(o, "Failed to read metadata: %s", err)
return time.Now()
}
// read mtime out of metadata if available
d, ok := o.meta[metaMtime]
if !ok {
// fs.FsDebug(o, "No metadata")
// fs.Debug(o, "No metadata")
return o.lastModified
}
modTime, err := swift.FloatStringToTime(d)
if err != nil {
fs.FsLog(o, "Failed to read mtime from object: %s", err)
fs.Log(o, "Failed to read mtime from object: %s", err)
return o.lastModified
}
return modTime
@ -337,14 +337,14 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
err := o.readMetaData()
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err)
fs.Log(o, "Failed to read metadata: %s", err)
return
}
o.meta[metaMtime] = swift.TimeToFloatString(modTime)
_, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta)
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err)
fs.Log(o, "Failed to update remote mtime: %s", err)
}
}
@ -366,4 +366,4 @@ func (o *FsObjectS3) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsS3{}
var _ fs.FsObject = &FsObjectS3{}
var _ fs.Object = &FsObjectS3{}

28
swift/fs.go
View File

@ -114,7 +114,7 @@ func NewFs(path string) (fs.Fs, error) {
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsObject {
func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.Object {
fs := &FsObjectSwift{
swift: f,
remote: remote,
@ -135,13 +135,13 @@ func (f *FsSwift) NewFsObjectWithInfo(remote string, info *swift.Object) fs.FsOb
// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.FsObject {
func (f *FsSwift) NewFsObject(remote string) fs.Object {
return f.NewFsObjectWithInfo(remote, nil)
}
// Walk the path returning a channel of FsObjects
func (f *FsSwift) List() fs.FsObjectsChan {
out := make(fs.FsObjectsChan, fs.Config.Checkers)
func (f *FsSwift) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() {
// FIXME use a smaller limit?
err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
@ -166,8 +166,8 @@ func (f *FsSwift) List() fs.FsObjectsChan {
}
// Lists the containers
func (f *FsSwift) ListDir() fs.FsDirChan {
out := make(fs.FsDirChan, fs.Config.Checkers)
func (f *FsSwift) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers)
go func() {
defer close(out)
containers, err := f.c.ContainersAll(nil)
@ -176,7 +176,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
log.Printf("Couldn't list containers: %s", err)
} else {
for _, container := range containers {
out <- &fs.FsDir{
out <- &fs.Dir{
Name: container.Name,
Bytes: container.Bytes,
Count: container.Count,
@ -192,7 +192,7 @@ func (f *FsSwift) ListDir() fs.FsDirChan {
// Copy the reader in to the new object which is returned
//
// The new object may have been created
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.FsObject, error) {
func (f *FsSwift) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction
fs := &FsObjectSwift{swift: f, remote: remote}
@ -246,7 +246,7 @@ func (o *FsObjectSwift) readMetaData() (err error) {
}
info, h, err := o.swift.c.Object(o.swift.container, o.remote)
if err != nil {
fs.FsDebug(o, "Failed to read info: %s", err)
fs.Debug(o, "Failed to read info: %s", err)
return err
}
meta := h.ObjectMetadata()
@ -263,12 +263,12 @@ func (o *FsObjectSwift) readMetaData() (err error) {
func (o *FsObjectSwift) ModTime() time.Time {
err := o.readMetaData()
if err != nil {
// fs.FsLog(o, "Failed to read metadata: %s", err)
// fs.Log(o, "Failed to read metadata: %s", err)
return o.info.LastModified
}
modTime, err := o.meta.GetModTime()
if err != nil {
// fs.FsLog(o, "Failed to read mtime from object: %s", err)
// fs.Log(o, "Failed to read mtime from object: %s", err)
return o.info.LastModified
}
return modTime
@ -279,14 +279,14 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
err := o.readMetaData()
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to read metadata: %s", err)
fs.Log(o, "Failed to read metadata: %s", err)
return
}
o.meta.SetModTime(modTime)
err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders())
if err != nil {
fs.Stats.Error()
fs.FsLog(o, "Failed to update remote mtime: %s", err)
fs.Log(o, "Failed to update remote mtime: %s", err)
}
}
@ -308,4 +308,4 @@ func (o *FsObjectSwift) Remove() error {
// Check the interfaces are satisfied
var _ fs.Fs = &FsSwift{}
var _ fs.FsObject = &FsObjectSwift{}
var _ fs.Object = &FsObjectSwift{}