diff --git a/Makefile b/Makefile index 715bf6344..8e941e592 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ rclone: test: rclone go test ./... - rclonetest/test.sh + fs/test_all.sh doc: rclone.1 README.html README.txt diff --git a/fs/operations.go b/fs/operations.go index ab78a7654..d4e7612c4 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -4,6 +4,7 @@ package fs import ( "fmt" + "io" "sync" ) @@ -430,9 +431,9 @@ func ListFn(f Fs, fn func(Object)) error { // Shows size and path // // Lists in parallel which may get them out of order -func List(f Fs) error { +func List(f Fs, w io.Writer) error { return ListFn(f, func(o Object) { - fmt.Printf("%9d %s\n", o.Size(), o.Remote()) + fmt.Fprintf(w, "%9d %s\n", o.Size(), o.Remote()) }) } @@ -441,12 +442,12 @@ func List(f Fs) error { // Shows size, mod time and path // // Lists in parallel which may get them out of order -func ListLong(f Fs) error { +func ListLong(f Fs, w io.Writer) error { return ListFn(f, func(o Object) { Stats.Checking(o) modTime := o.ModTime() Stats.DoneChecking(o) - fmt.Printf("%9d %19s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.00000000"), o.Remote()) + fmt.Fprintf(w, "%9d %s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.000000000"), o.Remote()) }) } @@ -455,7 +456,7 @@ func ListLong(f Fs) error { // Produces the same output as the md5sum command // // Lists in parallel which may get them out of order -func Md5sum(f Fs) error { +func Md5sum(f Fs, w io.Writer) error { return ListFn(f, func(o Object) { Stats.Checking(o) md5sum, err := o.Md5sum() @@ -464,14 +465,14 @@ func Md5sum(f Fs) error { Debug(o, "Failed to read MD5: %v", err) md5sum = "UNKNOWN" } - fmt.Printf("%32s %s\n", md5sum, o.Remote()) + fmt.Fprintf(w, "%32s %s\n", md5sum, o.Remote()) }) } // List the directories/buckets/containers in the Fs to stdout -func ListDir(f Fs) error { +func ListDir(f Fs, w io.Writer) error { for dir := range f.ListDir() { - fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name) + fmt.Fprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name) } return nil } diff --git a/fs/operations_test.go b/fs/operations_test.go new file mode 100644 index 000000000..7c2cbaba8 --- /dev/null +++ b/fs/operations_test.go @@ -0,0 +1,332 @@ +// Test rclone by doing real transactions to a storage provider to and +// from the local disk + +package fs_test + +import ( + "bytes" + "flag" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "strings" + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest" + + // Active file systems + _ "github.com/ncw/rclone/drive" + _ "github.com/ncw/rclone/dropbox" + _ "github.com/ncw/rclone/googlecloudstorage" + _ "github.com/ncw/rclone/local" + _ "github.com/ncw/rclone/s3" + _ "github.com/ncw/rclone/swift" +) + +// Globals +var ( + localName, remoteName string + flocal, fremote fs.Fs + RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem") + SubDir = flag.Bool("subdir", false, "Set to test with a sub directory") + finalise func() +) + +// Write a file +func WriteFile(filePath, content string, t time.Time) { + // FIXME make directories? + filePath = path.Join(localName, filePath) + dirPath := path.Dir(filePath) + err := os.MkdirAll(dirPath, 0770) + if err != nil { + log.Fatalf("Failed to make directories %q: %v", dirPath, err) + } + err = ioutil.WriteFile(filePath, []byte(content), 0600) + if err != nil { + log.Fatalf("Failed to write file %q: %v", filePath, err) + } + err = os.Chtimes(filePath, t, t) + if err != nil { + log.Fatalf("Failed to chtimes file %q: %v", filePath, err) + } +} + +var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") +var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") +var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") + +func TestInit(t *testing.T) { + fs.LoadConfig() + fs.Config.Verbose = false + fs.Config.Quiet = true + var err error + fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir) + if err != nil { + t.Fatalf("Failed to open remote %q: %v", *RemoteName, err) + } + t.Logf("Testing with remote %v", fremote) + + localName, err = ioutil.TempDir("", "rclone") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Logf("Testing with local %q", localName) + flocal, err = fs.NewFs(localName) + if err != nil { + t.Fatalf("Failed to make %q: %v", remoteName, err) + } + +} +func TestCalculateModifyWindow(t *testing.T) { + fs.CalculateModifyWindow(fremote, flocal) +} + +func TestMkdir(t *testing.T) { + fstest.TestMkdir(t, fremote) +} + +// Check dry run is working +func TestCopyWithDryRun(t *testing.T) { + WriteFile("sub dir/hello world", "hello world", t1) + + fs.Config.DryRun = true + err := fs.Sync(fremote, flocal, false) + fs.Config.DryRun = false + if err != nil { + t.Fatalf("Copy failed: %v", err) + } + + items := []fstest.Item{ + {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, + } + + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, []fstest.Item{}) +} + +// Now without dry run +func TestCopy(t *testing.T) { + err := fs.Sync(fremote, flocal, false) + if err != nil { + t.Fatalf("Copy failed: %v", err) + } + + items := []fstest.Item{ + {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, + } + + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +func TestLsd(t *testing.T) { + var buf bytes.Buffer + err := fs.ListDir(fremote, &buf) + if err != nil { + t.Fatalf("ListDir failed: %v", err) + } + res := buf.String() + if !strings.Contains(res, "sub dir\n") { + t.Fatalf("Result wrong %q", res) + } +} + +// Now delete the local file and download it +func TestCopyAfterDelete(t *testing.T) { + err := os.Remove(localName + "/sub dir/hello world") + if err != nil { + t.Fatalf("Remove failed: %v", err) + } + + items := []fstest.Item{ + {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, + } + fstest.CheckListing(t, flocal, []fstest.Item{}) + fstest.CheckListing(t, fremote, items) +} + +func TestCopyRedownload(t *testing.T) { + err := fs.Sync(flocal, fremote, false) + if err != nil { + t.Fatalf("Copy failed: %v", err) + } + + items := []fstest.Item{ + {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, + } + fstest.CheckListingWithPrecision(t, flocal, items, fremote.Precision()) + fstest.CheckListing(t, fremote, items) + + // Clean the directory + cleanTempDir(t) +} + +func TestSyncAfterChangingModtimeOnly(t *testing.T) { + WriteFile("empty space", "", t1) + + err := os.Chtimes(localName+"/empty space", t2, t2) + if err != nil { + t.Fatalf("Chtimes failed: %v", err) + } + err = fs.Sync(fremote, flocal, true) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +func TestSyncAfterAddingAFile(t *testing.T) { + WriteFile("potato", "------------------------------------------------------------", t3) + err := fs.Sync(fremote, flocal, true) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +func TestSyncAfterChangingFilesSizeOnly(t *testing.T) { + WriteFile("potato", "smaller but same date", t3) + err := fs.Sync(fremote, flocal, true) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +// Sync after changing a file's contents, modtime but not length +func TestSyncAfterChangingContentsOnly(t *testing.T) { + WriteFile("potato", "SMALLER BUT SAME DATE", t2) + err := fs.Sync(fremote, flocal, true) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +// Sync after removing a file and adding a file --dry-run +func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) { + WriteFile("potato2", "------------------------------------------------------------", t1) + err := os.Remove(localName + "/potato") + if err != nil { + t.Fatalf("Remove failed: %v", err) + } + fs.Config.DryRun = true + err = fs.Sync(fremote, flocal, true) + fs.Config.DryRun = false + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + + before := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, before) +} + +// Sync after removing a file and adding a file +func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) { + err := fs.Sync(fremote, flocal, true) + if err != nil { + t.Fatalf("Sync failed: %v", err) + } + items := []fstest.Item{ + {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, + {Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, + } + fstest.CheckListing(t, flocal, items) + fstest.CheckListing(t, fremote, items) +} + +func TestLs(t *testing.T) { + var buf bytes.Buffer + err := fs.List(fremote, &buf) + if err != nil { + t.Fatalf("List failed: %v", err) + } + res := buf.String() + if !strings.Contains(res, " 0 empty space\n") { + t.Errorf("empty space missing: %q", res) + } + if !strings.Contains(res, " 60 potato2\n") { + t.Errorf("potato2 missing: %q", res) + } +} + +func TestLsLong(t *testing.T) { + var buf bytes.Buffer + err := fs.ListLong(fremote, &buf) + if err != nil { + t.Fatalf("List failed: %v", err) + } + res := buf.String() + m1 := regexp.MustCompile(`(?m)^ 0 2011-12-25 12:59:59\.\d{9} empty space$`) + if !m1.MatchString(res) { + t.Errorf("empty space missing: %q", res) + } + m2 := regexp.MustCompile(`(?m)^ 60 2001-02-03 04:05:06\.\d{9} potato2$`) + if !m2.MatchString(res) { + t.Errorf("potato2 missing: %q", res) + } +} + +func TestMd5sum(t *testing.T) { + var buf bytes.Buffer + err := fs.Md5sum(fremote, &buf) + if err != nil { + t.Fatalf("List failed: %v", err) + } + res := buf.String() + if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e empty space\n") { + t.Errorf("empty space missing: %q", res) + } + if !strings.Contains(res, "6548b156ea68a4e003e786df99eee76 potato2\n") { + t.Errorf("potato2 missing: %q", res) + } +} + +func TestCheck(t *testing.T) { +} + +// Clean the temporary directory +func cleanTempDir(t *testing.T) { + t.Logf("Cleaning temporary directory: %q", localName) + err := os.RemoveAll(localName) + if err != nil { + t.Logf("Failed to remove %q: %v", localName, err) + } +} + +func TestFinalise(t *testing.T) { + finalise() + + cleanTempDir(t) +} diff --git a/fs/test_all.sh b/fs/test_all.sh new file mode 100755 index 000000000..d20e0cbdb --- /dev/null +++ b/fs/test_all.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +go install + +REMOTES=" +TestSwift: +TestS3: +TestDrive: +TestGoogleCloudStorage: +TestDropbox: +" + +function test_remote { + args=$@ + echo "@go test $args" + go test $args || { + echo "*** test $args FAILED ***" + exit 1 + } +} + +test_remote +test_remote --subdir +for remote in $REMOTES; do + test_remote --remote $remote + test_remote --remote $remote --subdir +done + +echo "All OK" diff --git a/fstest/fstest.go b/fstest/fstest.go index 4d9f40d0d..ad4c0e0e8 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -10,13 +10,12 @@ import ( "math/rand" "os" "strings" + "testing" "time" "github.com/ncw/rclone/fs" ) -var Fatalf = log.Fatalf - // Seed the random number generator func init() { rand.Seed(time.Now().UnixNano()) @@ -32,30 +31,29 @@ type Item struct { } // check the mod time to the given precision -func (i *Item) CheckModTime(obj fs.Object, modTime time.Time) { +func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) { dt := modTime.Sub(i.ModTime) - precision := obj.Fs().Precision() if dt >= precision || dt <= -precision { - Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, precision, modTime, i.ModTime) + t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", obj.Remote(), dt, precision, modTime, i.ModTime, precision) } } -func (i *Item) Check(obj fs.Object) { +func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) { if obj == nil { - Fatalf("Object is nil") + t.Fatalf("Object is nil") } // Check attributes Md5sum, err := obj.Md5sum() if err != nil { - Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err) + t.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err) } if i.Md5sum != Md5sum { - Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum) + t.Errorf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum) } if i.Size != obj.Size() { - Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size()) + t.Errorf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size()) } - i.CheckModTime(obj, obj.ModTime()) + i.CheckModTime(t, obj, obj.ModTime(), precision) } // Represents all items for checking @@ -78,39 +76,45 @@ func NewItems(items []Item) *Items { } // Check off an item -func (is *Items) Find(obj fs.Object) { +func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) { i, ok := is.byName[obj.Remote()] if !ok { - Fatalf("Unexpected file %q", obj.Remote()) + t.Errorf("Unexpected file %q", obj.Remote()) } delete(is.byName, obj.Remote()) - i.Check(obj) + i.Check(t, obj, precision) } // Check all done -func (is *Items) Done() { +func (is *Items) Done(t *testing.T) { if len(is.byName) != 0 { for name := range is.byName { log.Printf("Not found %q", name) } - Fatalf("%d objects not found", len(is.byName)) + t.Errorf("%d objects not found", len(is.byName)) } } // Checks the fs to see if it has the expected contents -func CheckListing(f fs.Fs, items []Item) { +func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { is := NewItems(items) for obj := range f.List() { - is.Find(obj) + is.Find(t, obj, precision) } - is.Done() + is.Done(t) +} + +// Checks the fs to see if it has the expected contents +func CheckListing(t *testing.T, f fs.Fs, items []Item) { + precision := f.Precision() + CheckListingWithPrecision(t, f, items, precision) } // Parse a time string or explode func Time(timeString string) time.Time { t, err := time.Parse(time.RFC3339Nano, timeString) if err != nil { - Fatalf("Failed to parse time %q: %v", timeString, err) + log.Fatalf("Failed to parse time %q: %v", timeString, err) } return t } @@ -197,25 +201,25 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) { return remote, finalise, nil } -func TestMkdir(remote fs.Fs) { +func TestMkdir(t *testing.T, remote fs.Fs) { err := fs.Mkdir(remote) if err != nil { - Fatalf("Mkdir failed: %v", err) + t.Fatalf("Mkdir failed: %v", err) } - CheckListing(remote, []Item{}) + CheckListing(t, remote, []Item{}) } -func TestPurge(remote fs.Fs) { +func TestPurge(t *testing.T, remote fs.Fs) { err := fs.Purge(remote) if err != nil { - Fatalf("Purge failed: %v", err) + t.Fatalf("Purge failed: %v", err) } - CheckListing(remote, []Item{}) + CheckListing(t, remote, []Item{}) } -func TestRmdir(remote fs.Fs) { +func TestRmdir(t *testing.T, remote fs.Fs) { err := fs.Rmdir(remote) if err != nil { - Fatalf("Rmdir failed: %v", err) + t.Fatalf("Rmdir failed: %v", err) } } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 330fdabf3..95120d81d 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1,8 +1,6 @@ // Generic tests for testing the Fs and Object interfaces package fstests -// FIXME need to check the limited file system - import ( "bytes" "crypto/md5" @@ -58,12 +56,10 @@ func TestInit(t *testing.T) { if err != nil { t.Fatalf("Couldn't start FS: %v", err) } - fstest.Fatalf = t.Fatalf - fstest.TestMkdir(remote) + fstest.TestMkdir(t, remote) } func skipIfNotOk(t *testing.T) { - fstest.Fatalf = t.Fatalf if remote == nil { t.Skip("FS not configured") } @@ -88,7 +84,7 @@ type TestFile struct { func TestFsRmdirEmpty(t *testing.T) { skipIfNotOk(t) - fstest.TestRmdir(remote) + fstest.TestRmdir(t, remote) } func TestFsRmdirNotFound(t *testing.T) { @@ -101,13 +97,13 @@ func TestFsRmdirNotFound(t *testing.T) { func TestFsMkdir(t *testing.T) { skipIfNotOk(t) - fstest.TestMkdir(remote) - fstest.TestMkdir(remote) + fstest.TestMkdir(t, remote) + fstest.TestMkdir(t, remote) } func TestFsListEmpty(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(remote, []fstest.Item{}) + fstest.CheckListing(t, remote, []fstest.Item{}) } func TestFsListDirEmpty(t *testing.T) { @@ -143,10 +139,10 @@ func testPut(t *testing.T, file *fstest.Item) { t.Fatal("Put error", err) } file.Md5sum = hex.EncodeToString(hash.Sum(nil)) - file.Check(obj) + file.Check(t, obj, remote.Precision()) // Re-read the object and check again obj = findObject(t, file.Path) - file.Check(obj) + file.Check(t, obj, remote.Precision()) } func TestFsPutFile1(t *testing.T) { @@ -231,18 +227,18 @@ func TestFsListRoot(t *testing.T) { func TestFsListFile1(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) } func TestFsNewFsObject(t *testing.T) { skipIfNotOk(t) obj := findObject(t, file1.Path) - file1.Check(obj) + file1.Check(t, obj, remote.Precision()) } func TestFsListFile1and2(t *testing.T) { skipIfNotOk(t) - fstest.CheckListing(remote, []fstest.Item{file1, file2}) + fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) } func TestFsRmdirFull(t *testing.T) { @@ -307,7 +303,7 @@ func TestObjectMd5sum(t *testing.T) { func TestObjectModTime(t *testing.T) { skipIfNotOk(t) obj := findObject(t, file1.Path) - file1.CheckModTime(obj, obj.ModTime()) + file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision()) } func TestObjectSetModTime(t *testing.T) { @@ -316,7 +312,7 @@ func TestObjectSetModTime(t *testing.T) { obj := findObject(t, file1.Path) obj.SetModTime(newModTime) file1.ModTime = newModTime - file1.CheckModTime(obj, newModTime) + file1.CheckModTime(t, obj, newModTime, remote.Precision()) // And make a new object and read it from there too TestObjectModTime(t) } @@ -367,10 +363,10 @@ func TestObjectUpdate(t *testing.T) { t.Fatal("Update error", err) } file1.Md5sum = hex.EncodeToString(hash.Sum(nil)) - file1.Check(obj) + file1.Check(t, obj, remote.Precision()) // Re-read the object and check again obj = findObject(t, file1.Path) - file1.Check(obj) + file1.Check(t, obj, remote.Precision()) } func TestObjectStorable(t *testing.T) { @@ -390,7 +386,7 @@ func TestLimitedFs(t *testing.T) { if err != nil { t.Fatal("Failed to make remote %q: %v", remoteName, err) } - fstest.CheckListing(fileRemote, []fstest.Item{file2Copy}) + fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy}) _, ok := fileRemote.(*fs.Limited) if !ok { t.Errorf("%v is not a fs.Limited", fileRemote) @@ -404,7 +400,7 @@ func TestLimitedFsNotFound(t *testing.T) { if err != nil { t.Fatal("Failed to make remote %q: %v", remoteName, err) } - fstest.CheckListing(fileRemote, []fstest.Item{}) + fstest.CheckListing(t, fileRemote, []fstest.Item{}) _, ok := fileRemote.(*fs.Limited) if ok { t.Errorf("%v is is a fs.Limited", fileRemote) @@ -418,12 +414,12 @@ func TestObjectRemove(t *testing.T) { if err != nil { t.Fatal("Remove error", err) } - fstest.CheckListing(remote, []fstest.Item{file2}) + fstest.CheckListing(t, remote, []fstest.Item{file2}) } func TestObjectPurge(t *testing.T) { skipIfNotOk(t) - fstest.TestPurge(remote) + fstest.TestPurge(t, remote) err := fs.Purge(remote) if err == nil { t.Fatal("Expecting error after on second purge") diff --git a/notes.txt b/notes.txt index ba3b8b66f..96f1f9e96 100644 --- a/notes.txt +++ b/notes.txt @@ -3,14 +3,13 @@ Change lsd command so it doesn't show -1 * Make test? Put the TestRemote names into the Fs description -Make rclonetest use the TestRemote name automatically -Put rclonetest back into rclone as tests - * defaults to using local remote - * but could pass another in with a flag +Make test_all.sh use the TestRemote name automatically Run errcheck and go vet in the make file .. Also race detector? +Get rid of Storable? + Write developer manual Todo diff --git a/rclone.go b/rclone.go index 9b290861d..6f29b939f 100644 --- a/rclone.go +++ b/rclone.go @@ -96,7 +96,7 @@ var Commands = []Command{ Help: ` List all the objects in the the path with size and path.`, Run: func(fdst, fsrc fs.Fs) { - err := fs.List(fdst) + err := fs.List(fdst, os.Stdout) if err != nil { log.Fatalf("Failed to list: %v", err) } @@ -110,7 +110,7 @@ var Commands = []Command{ Help: ` List all directories/containers/buckets in the the path.`, Run: func(fdst, fsrc fs.Fs) { - err := fs.ListDir(fdst) + err := fs.ListDir(fdst, os.Stdout) if err != nil { log.Fatalf("Failed to listdir: %v", err) } @@ -124,7 +124,7 @@ var Commands = []Command{ Help: ` List all the objects in the the path with modification time, size and path.`, Run: func(fdst, fsrc fs.Fs) { - err := fs.ListLong(fdst) + err := fs.ListLong(fdst, os.Stdout) if err != nil { log.Fatalf("Failed to list long: %v", err) } @@ -138,7 +138,7 @@ var Commands = []Command{ Help: ` Produces an md5sum file for all the objects in the path.`, Run: func(fdst, fsrc fs.Fs) { - err := fs.Md5sum(fdst) + err := fs.Md5sum(fdst, os.Stdout) if err != nil { log.Fatalf("Failed to list: %v", err) } diff --git a/rclonetest/rclonetest.go b/rclonetest/rclonetest.go deleted file mode 100644 index 2876b19af..000000000 --- a/rclonetest/rclonetest.go +++ /dev/null @@ -1,290 +0,0 @@ -// Test rclone by doing real transactions to a storage provider to and -// from the local disk - -package main - -import ( - "fmt" - "io/ioutil" - "log" - "os" - "path" - "time" - - "github.com/ncw/rclone/fs" - "github.com/ncw/rclone/fstest" - "github.com/ogier/pflag" - - // Active file systems - _ "github.com/ncw/rclone/drive" - _ "github.com/ncw/rclone/dropbox" - _ "github.com/ncw/rclone/googlecloudstorage" - _ "github.com/ncw/rclone/local" - _ "github.com/ncw/rclone/s3" - _ "github.com/ncw/rclone/swift" -) - -// Globals -var ( - localName, remoteName string - version = pflag.BoolP("version", "V", false, "Print the version number") - subDir = pflag.BoolP("subdir", "S", false, "Test with a sub directory") -) - -// Write a file -func WriteFile(filePath, content string, t time.Time) { - // FIXME make directories? - filePath = path.Join(localName, filePath) - dirPath := path.Dir(filePath) - err := os.MkdirAll(dirPath, 0770) - if err != nil { - log.Fatalf("Failed to make directories %q: %v", dirPath, err) - } - err = ioutil.WriteFile(filePath, []byte(content), 0600) - if err != nil { - log.Fatalf("Failed to write file %q: %v", filePath, err) - } - err = os.Chtimes(filePath, t, t) - if err != nil { - log.Fatalf("Failed to chtimes file %q: %v", filePath, err) - } -} - -var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") -var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") -var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") - -func TestCopy(flocal, fremote fs.Fs) { - WriteFile("sub dir/hello world", "hello world", t1) - - // Check dry run is working - log.Printf("Copy with --dry-run") - fs.Config.DryRun = true - err := fs.Sync(fremote, flocal, false) - fs.Config.DryRun = false - if err != nil { - log.Fatalf("Copy failed: %v", err) - } - - items := []fstest.Item{ - {Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"}, - } - - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, []fstest.Item{}) - - // Now without dry run - - log.Printf("Copy") - err = fs.Sync(fremote, flocal, false) - if err != nil { - log.Fatalf("Copy failed: %v", err) - } - - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // Now delete the local file and download it - - err = os.Remove(localName + "/sub dir/hello world") - if err != nil { - log.Fatalf("Remove failed: %v", err) - } - - fstest.CheckListing(flocal, []fstest.Item{}) - fstest.CheckListing(fremote, items) - - log.Printf("Copy - redownload") - err = fs.Sync(flocal, fremote, false) - if err != nil { - log.Fatalf("Copy failed: %v", err) - } - - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // Clean the directory - cleanTempDir() -} - -func TestSync(flocal, fremote fs.Fs) { - WriteFile("empty space", "", t1) - - log.Printf("Sync after changing file modtime only") - err := os.Chtimes(localName+"/empty space", t2, t2) - if err != nil { - log.Fatalf("Chtimes failed: %v", err) - } - err = fs.Sync(fremote, flocal, true) - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - items := []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // ------------------------------------------------------------ - - log.Printf("Sync after adding a file") - WriteFile("potato", "------------------------------------------------------------", t3) - err = fs.Sync(fremote, flocal, true) - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - items = []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - {Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // ------------------------------------------------------------ - - log.Printf("Sync after changing a file's size only") - WriteFile("potato", "smaller but same date", t3) - err = fs.Sync(fremote, flocal, true) - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - items = []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - {Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"}, - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // ------------------------------------------------------------ - - log.Printf("Sync after changing a file's contents, modtime but not length") - WriteFile("potato", "SMALLER BUT SAME DATE", t2) - err = fs.Sync(fremote, flocal, true) - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - items = []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) - - // ------------------------------------------------------------ - - log.Printf("Sync after removing a file and adding a file --dry-run") - WriteFile("potato2", "------------------------------------------------------------", t1) - err = os.Remove(localName + "/potato") - if err != nil { - log.Fatalf("Remove failed: %v", err) - } - fs.Config.DryRun = true - err = fs.Sync(fremote, flocal, true) - fs.Config.DryRun = false - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - - before := []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - {Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"}, - } - items = []fstest.Item{ - {Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"}, - {Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, before) - - log.Printf("Sync after removing a file and adding a file") - err = fs.Sync(fremote, flocal, true) - if err != nil { - log.Fatalf("Sync failed: %v", err) - } - fstest.CheckListing(flocal, items) - fstest.CheckListing(fremote, items) -} - -func TestLs(flocal, fremote fs.Fs) { - // Underlying List has been tested above, so we just make sure it runs - err := fs.List(fremote) - if err != nil { - log.Fatalf("List failed: %v", err) - } -} - -func TestLsd(flocal, fremote fs.Fs) { -} - -func TestCheck(flocal, fremote fs.Fs) { -} - -func syntaxError() { - fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s. - -Syntax: [options] remote: - -Need a remote: as argument. This will create a random container or -directory under it and perform tests on it, deleting it at the end. - -Options: - -`, fs.Version) - pflag.PrintDefaults() -} - -// Clean the temporary directory -func cleanTempDir() { - log.Printf("Cleaning temporary directory: %q", localName) - err := os.RemoveAll(localName) - if err != nil { - log.Printf("Failed to remove %q: %v", localName, err) - } -} - -func main() { - pflag.Usage = syntaxError - pflag.Parse() - if *version { - fmt.Printf("rclonetest %s\n", fs.Version) - os.Exit(0) - } - fs.LoadConfig() - args := pflag.Args() - - if len(args) != 1 { - syntaxError() - os.Exit(1) - } - - fremote, finalise, err := fstest.RandomRemote(args[0], *subDir) - if err != nil { - log.Fatalf("Failed to open remote %q: %v", args[0], err) - } - log.Printf("Testing with remote %v", fremote) - - localName, err = ioutil.TempDir("", "rclone") - if err != nil { - log.Fatalf("Failed to create temp dir: %v", err) - } - log.Printf("Testing with local %q", localName) - flocal, err := fs.NewFs(localName) - if err != nil { - log.Fatalf("Failed to make %q: %v", remoteName, err) - } - - fs.CalculateModifyWindow(fremote, flocal) - - fstest.TestMkdir(fremote) - TestCopy(flocal, fremote) - TestSync(flocal, fremote) - TestLs(flocal, fremote) - TestLsd(flocal, fremote) - TestCheck(flocal, fremote) - //TestRmdir(flocal, fremote) - - finalise() - - cleanTempDir() - log.Printf("Tests OK") -} diff --git a/rclonetest/test.sh b/rclonetest/test.sh deleted file mode 100755 index 740f3fec4..000000000 --- a/rclonetest/test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -go install - -REMOTES=" -TestSwift: -TestS3: -TestDrive: -TestGoogleCloudStorage: -TestDropbox: -/tmp/z -" - -function test_remote { - args=$@ - rclonetest $args || { - echo "*** rclonetest $args FAILED ***" - exit 1 - } -} - -for remote in $REMOTES; do - test_remote $remote - test_remote --subdir $remote -done