Move rclonetest into go tests for fs module

Makefile

@@ -8,7 +8,7 @@ rclone:
 
 test:	rclone
 	go test ./...
-	rclonetest/test.sh
+	fs/test_all.sh
 
 doc:	rclone.1 README.html README.txt
 

fs/operations.go

@@ -4,6 +4,7 @@ package fs
 
 import (
 	"fmt"
+	"io"
 	"sync"
 )
 
@@ -430,9 +431,9 @@ func ListFn(f Fs, fn func(Object)) error {
 // Shows size and path
 //
 // Lists in parallel which may get them out of order
-func List(f Fs) error {
+func List(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
-		fmt.Printf("%9d %s\n", o.Size(), o.Remote())
+		fmt.Fprintf(w, "%9d %s\n", o.Size(), o.Remote())
 	})
 }
 
@@ -441,12 +442,12 @@ func List(f Fs) error {
 // Shows size, mod time and path
 //
 // Lists in parallel which may get them out of order
-func ListLong(f Fs) error {
+func ListLong(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		modTime := o.ModTime()
 		Stats.DoneChecking(o)
-		fmt.Printf("%9d %19s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.00000000"), o.Remote())
+		fmt.Fprintf(w, "%9d %s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.000000000"), o.Remote())
 	})
 }
 
@@ -455,7 +456,7 @@ func ListLong(f Fs) error {
 // Produces the same output as the md5sum command
 //
 // Lists in parallel which may get them out of order
-func Md5sum(f Fs) error {
+func Md5sum(f Fs, w io.Writer) error {
 	return ListFn(f, func(o Object) {
 		Stats.Checking(o)
 		md5sum, err := o.Md5sum()
@@ -464,14 +465,14 @@ func Md5sum(f Fs) error {
 			Debug(o, "Failed to read MD5: %v", err)
 			md5sum = "UNKNOWN"
 		}
-		fmt.Printf("%32s  %s\n", md5sum, o.Remote())
+		fmt.Fprintf(w, "%32s  %s\n", md5sum, o.Remote())
 	})
 }
 
 // List the directories/buckets/containers in the Fs to stdout
-func ListDir(f Fs) error {
+func ListDir(f Fs, w io.Writer) error {
 	for dir := range f.ListDir() {
-		fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
+		fmt.Fprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
 	}
 	return nil
 }

fs/operations_test.go

@@ -0,0 +1,332 @@
+// Test rclone by doing real transactions to a storage provider to and
+// from the local disk
+
+package fs_test
+
+import (
+	"bytes"
+	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+
+	// Active file systems
+	_ "github.com/ncw/rclone/drive"
+	_ "github.com/ncw/rclone/dropbox"
+	_ "github.com/ncw/rclone/googlecloudstorage"
+	_ "github.com/ncw/rclone/local"
+	_ "github.com/ncw/rclone/s3"
+	_ "github.com/ncw/rclone/swift"
+)
+
+// Globals
+var (
+	localName, remoteName string
+	flocal, fremote       fs.Fs
+	RemoteName            = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
+	SubDir                = flag.Bool("subdir", false, "Set to test with a sub directory")
+	finalise              func()
+)
+
+// Write a file
+func WriteFile(filePath, content string, t time.Time) {
+	// FIXME make directories?
+	filePath = path.Join(localName, filePath)
+	dirPath := path.Dir(filePath)
+	err := os.MkdirAll(dirPath, 0770)
+	if err != nil {
+		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
+	}
+	err = ioutil.WriteFile(filePath, []byte(content), 0600)
+	if err != nil {
+		log.Fatalf("Failed to write file %q: %v", filePath, err)
+	}
+	err = os.Chtimes(filePath, t, t)
+	if err != nil {
+		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
+	}
+}
+
+var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
+var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
+var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
+
+func TestInit(t *testing.T) {
+	fs.LoadConfig()
+	fs.Config.Verbose = false
+	fs.Config.Quiet = true
+	var err error
+	fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
+	if err != nil {
+		t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
+	}
+	t.Logf("Testing with remote %v", fremote)
+
+	localName, err = ioutil.TempDir("", "rclone")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	t.Logf("Testing with local %q", localName)
+	flocal, err = fs.NewFs(localName)
+	if err != nil {
+		t.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+
+}
+func TestCalculateModifyWindow(t *testing.T) {
+	fs.CalculateModifyWindow(fremote, flocal)
+}
+
+func TestMkdir(t *testing.T) {
+	fstest.TestMkdir(t, fremote)
+}
+
+// Check dry run is working
+func TestCopyWithDryRun(t *testing.T) {
+	WriteFile("sub dir/hello world", "hello world", t1)
+
+	fs.Config.DryRun = true
+	err := fs.Sync(fremote, flocal, false)
+	fs.Config.DryRun = false
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, []fstest.Item{})
+}
+
+// Now without dry run
+func TestCopy(t *testing.T) {
+	err := fs.Sync(fremote, flocal, false)
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestLsd(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.ListDir(fremote, &buf)
+	if err != nil {
+		t.Fatalf("ListDir failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "sub dir\n") {
+		t.Fatalf("Result wrong %q", res)
+	}
+}
+
+// Now delete the local file and download it
+func TestCopyAfterDelete(t *testing.T) {
+	err := os.Remove(localName + "/sub dir/hello world")
+	if err != nil {
+		t.Fatalf("Remove failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+	fstest.CheckListing(t, flocal, []fstest.Item{})
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestCopyRedownload(t *testing.T) {
+	err := fs.Sync(flocal, fremote, false)
+	if err != nil {
+		t.Fatalf("Copy failed: %v", err)
+	}
+
+	items := []fstest.Item{
+		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
+	}
+	fstest.CheckListingWithPrecision(t, flocal, items, fremote.Precision())
+	fstest.CheckListing(t, fremote, items)
+
+	// Clean the directory
+	cleanTempDir(t)
+}
+
+func TestSyncAfterChangingModtimeOnly(t *testing.T) {
+	WriteFile("empty space", "", t1)
+
+	err := os.Chtimes(localName+"/empty space", t2, t2)
+	if err != nil {
+		t.Fatalf("Chtimes failed: %v", err)
+	}
+	err = fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestSyncAfterAddingAFile(t *testing.T) {
+	WriteFile("potato", "------------------------------------------------------------", t3)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
+	WriteFile("potato", "smaller but same date", t3)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+// Sync after changing a file's contents, modtime but not length
+func TestSyncAfterChangingContentsOnly(t *testing.T) {
+	WriteFile("potato", "SMALLER BUT SAME DATE", t2)
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+// Sync after removing a file and adding a file --dry-run
+func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
+	WriteFile("potato2", "------------------------------------------------------------", t1)
+	err := os.Remove(localName + "/potato")
+	if err != nil {
+		t.Fatalf("Remove failed: %v", err)
+	}
+	fs.Config.DryRun = true
+	err = fs.Sync(fremote, flocal, true)
+	fs.Config.DryRun = false
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+
+	before := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, before)
+}
+
+// Sync after removing a file and adding a file
+func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
+	err := fs.Sync(fremote, flocal, true)
+	if err != nil {
+		t.Fatalf("Sync failed: %v", err)
+	}
+	items := []fstest.Item{
+		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
+		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
+	}
+	fstest.CheckListing(t, flocal, items)
+	fstest.CheckListing(t, fremote, items)
+}
+
+func TestLs(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.List(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "        0 empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "       60 potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestLsLong(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.ListLong(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	m1 := regexp.MustCompile(`(?m)^        0 2011-12-25 12:59:59\.\d{9} empty space$`)
+	if !m1.MatchString(res) {
+		t.Errorf("empty space missing: %q", res)
+	}
+	m2 := regexp.MustCompile(`(?m)^       60 2001-02-03 04:05:06\.\d{9} potato2$`)
+	if !m2.MatchString(res) {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestMd5sum(t *testing.T) {
+	var buf bytes.Buffer
+	err := fs.Md5sum(fremote, &buf)
+	if err != nil {
+		t.Fatalf("List failed: %v", err)
+	}
+	res := buf.String()
+	if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e  empty space\n") {
+		t.Errorf("empty space missing: %q", res)
+	}
+	if !strings.Contains(res, "6548b156ea68a4e003e786df99eee76  potato2\n") {
+		t.Errorf("potato2 missing: %q", res)
+	}
+}
+
+func TestCheck(t *testing.T) {
+}
+
+// Clean the temporary directory
+func cleanTempDir(t *testing.T) {
+	t.Logf("Cleaning temporary directory: %q", localName)
+	err := os.RemoveAll(localName)
+	if err != nil {
+		t.Logf("Failed to remove %q: %v", localName, err)
+	}
+}
+
+func TestFinalise(t *testing.T) {
+	finalise()
+
+	cleanTempDir(t)
+}

fs/test_all.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+go install
+
+REMOTES="
+TestSwift:
+TestS3:
+TestDrive:
+TestGoogleCloudStorage:
+TestDropbox:
+"
+
+function test_remote {
+    args=$@
+    echo "@go test $args"
+    go test $args || {
+        echo "*** test $args FAILED ***"
+        exit 1
+    }
+}
+
+test_remote
+test_remote --subdir
+for remote in $REMOTES; do
+    test_remote --remote $remote
+    test_remote --remote $remote --subdir
+done
+
+echo "All OK"

fstest/fstest.go

@@ -10,13 +10,12 @@ import (
 	"math/rand"
 	"os"
 	"strings"
+	"testing"
 	"time"
 
 	"github.com/ncw/rclone/fs"
 )
 
-var Fatalf = log.Fatalf
-
 // Seed the random number generator
 func init() {
 	rand.Seed(time.Now().UnixNano())
@@ -32,30 +31,29 @@ type Item struct {
 }
 
 // check the mod time to the given precision
-func (i *Item) CheckModTime(obj fs.Object, modTime time.Time) {
+func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
 	dt := modTime.Sub(i.ModTime)
-	precision := obj.Fs().Precision()
 	if dt >= precision || dt <= -precision {
-		Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, precision, modTime, i.ModTime)
+		t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", obj.Remote(), dt, precision, modTime, i.ModTime, precision)
 	}
 }
 
-func (i *Item) Check(obj fs.Object) {
+func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
 	if obj == nil {
-		Fatalf("Object is nil")
+		t.Fatalf("Object is nil")
 	}
 	// Check attributes
 	Md5sum, err := obj.Md5sum()
 	if err != nil {
-		Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
+		t.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
 	}
 	if i.Md5sum != Md5sum {
-		Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
+		t.Errorf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
 	}
 	if i.Size != obj.Size() {
-		Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
+		t.Errorf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
 	}
-	i.CheckModTime(obj, obj.ModTime())
+	i.CheckModTime(t, obj, obj.ModTime(), precision)
 }
 
 // Represents all items for checking
@@ -78,39 +76,45 @@ func NewItems(items []Item) *Items {
 }
 
 // Check off an item
-func (is *Items) Find(obj fs.Object) {
+func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
 	i, ok := is.byName[obj.Remote()]
 	if !ok {
-		Fatalf("Unexpected file %q", obj.Remote())
+		t.Errorf("Unexpected file %q", obj.Remote())
 	}
 	delete(is.byName, obj.Remote())
-	i.Check(obj)
+	i.Check(t, obj, precision)
 }
 
 // Check all done
-func (is *Items) Done() {
+func (is *Items) Done(t *testing.T) {
 	if len(is.byName) != 0 {
 		for name := range is.byName {
 			log.Printf("Not found %q", name)
 		}
-		Fatalf("%d objects not found", len(is.byName))
+		t.Errorf("%d objects not found", len(is.byName))
 	}
 }
 
 // Checks the fs to see if it has the expected contents
-func CheckListing(f fs.Fs, items []Item) {
+func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
 	is := NewItems(items)
 	for obj := range f.List() {
-		is.Find(obj)
+		is.Find(t, obj, precision)
 	}
-	is.Done()
+	is.Done(t)
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListing(t *testing.T, f fs.Fs, items []Item) {
+	precision := f.Precision()
+	CheckListingWithPrecision(t, f, items, precision)
 }
 
 // Parse a time string or explode
 func Time(timeString string) time.Time {
 	t, err := time.Parse(time.RFC3339Nano, timeString)
 	if err != nil {
-		Fatalf("Failed to parse time %q: %v", timeString, err)
+		log.Fatalf("Failed to parse time %q: %v", timeString, err)
 	}
 	return t
 }
@@ -197,25 +201,25 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
 	return remote, finalise, nil
 }
 
-func TestMkdir(remote fs.Fs) {
+func TestMkdir(t *testing.T, remote fs.Fs) {
 	err := fs.Mkdir(remote)
 	if err != nil {
-		Fatalf("Mkdir failed: %v", err)
+		t.Fatalf("Mkdir failed: %v", err)
 	}
-	CheckListing(remote, []Item{})
+	CheckListing(t, remote, []Item{})
 }
 
-func TestPurge(remote fs.Fs) {
+func TestPurge(t *testing.T, remote fs.Fs) {
 	err := fs.Purge(remote)
 	if err != nil {
-		Fatalf("Purge failed: %v", err)
+		t.Fatalf("Purge failed: %v", err)
 	}
-	CheckListing(remote, []Item{})
+	CheckListing(t, remote, []Item{})
 }
 
-func TestRmdir(remote fs.Fs) {
+func TestRmdir(t *testing.T, remote fs.Fs) {
 	err := fs.Rmdir(remote)
 	if err != nil {
-		Fatalf("Rmdir failed: %v", err)
+		t.Fatalf("Rmdir failed: %v", err)
 	}
 }

fstest/fstests/fstests.go

@@ -1,8 +1,6 @@
 // Generic tests for testing the Fs and Object interfaces
 package fstests
 
-// FIXME need to check the limited file system
-
 import (
 	"bytes"
 	"crypto/md5"
@@ -58,12 +56,10 @@ func TestInit(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Couldn't start FS: %v", err)
 	}
-	fstest.Fatalf = t.Fatalf
+	fstest.TestMkdir(t, remote)
-	fstest.TestMkdir(remote)
 }
 
 func skipIfNotOk(t *testing.T) {
-	fstest.Fatalf = t.Fatalf
 	if remote == nil {
 		t.Skip("FS not configured")
 	}
@@ -88,7 +84,7 @@ type TestFile struct {
 
 func TestFsRmdirEmpty(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.TestRmdir(remote)
+	fstest.TestRmdir(t, remote)
 }
 
 func TestFsRmdirNotFound(t *testing.T) {
@@ -101,13 +97,13 @@ func TestFsRmdirNotFound(t *testing.T) {
 
 func TestFsMkdir(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.TestMkdir(remote)
+	fstest.TestMkdir(t, remote)
-	fstest.TestMkdir(remote)
+	fstest.TestMkdir(t, remote)
 }
 
 func TestFsListEmpty(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.CheckListing(remote, []fstest.Item{})
+	fstest.CheckListing(t, remote, []fstest.Item{})
 }
 
 func TestFsListDirEmpty(t *testing.T) {
@@ -143,10 +139,10 @@ func testPut(t *testing.T, file *fstest.Item) {
 		t.Fatal("Put error", err)
 	}
 	file.Md5sum = hex.EncodeToString(hash.Sum(nil))
-	file.Check(obj)
+	file.Check(t, obj, remote.Precision())
 	// Re-read the object and check again
 	obj = findObject(t, file.Path)
-	file.Check(obj)
+	file.Check(t, obj, remote.Precision())
 }
 
 func TestFsPutFile1(t *testing.T) {
@@ -231,18 +227,18 @@ func TestFsListRoot(t *testing.T) {
 
 func TestFsListFile1(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
 }
 
 func TestFsNewFsObject(t *testing.T) {
 	skipIfNotOk(t)
 	obj := findObject(t, file1.Path)
-	file1.Check(obj)
+	file1.Check(t, obj, remote.Precision())
 }
 
 func TestFsListFile1and2(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+	fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
 }
 
 func TestFsRmdirFull(t *testing.T) {
@@ -307,7 +303,7 @@ func TestObjectMd5sum(t *testing.T) {
 func TestObjectModTime(t *testing.T) {
 	skipIfNotOk(t)
 	obj := findObject(t, file1.Path)
-	file1.CheckModTime(obj, obj.ModTime())
+	file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
 }
 
 func TestObjectSetModTime(t *testing.T) {
@@ -316,7 +312,7 @@ func TestObjectSetModTime(t *testing.T) {
 	obj := findObject(t, file1.Path)
 	obj.SetModTime(newModTime)
 	file1.ModTime = newModTime
-	file1.CheckModTime(obj, newModTime)
+	file1.CheckModTime(t, obj, newModTime, remote.Precision())
 	// And make a new object and read it from there too
 	TestObjectModTime(t)
 }
@@ -367,10 +363,10 @@ func TestObjectUpdate(t *testing.T) {
 		t.Fatal("Update error", err)
 	}
 	file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
-	file1.Check(obj)
+	file1.Check(t, obj, remote.Precision())
 	// Re-read the object and check again
 	obj = findObject(t, file1.Path)
-	file1.Check(obj)
+	file1.Check(t, obj, remote.Precision())
 }
 
 func TestObjectStorable(t *testing.T) {
@@ -390,7 +386,7 @@ func TestLimitedFs(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to make remote %q: %v", remoteName, err)
 	}
-	fstest.CheckListing(fileRemote, []fstest.Item{file2Copy})
+	fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
 	_, ok := fileRemote.(*fs.Limited)
 	if !ok {
 		t.Errorf("%v is not a fs.Limited", fileRemote)
@@ -404,7 +400,7 @@ func TestLimitedFsNotFound(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to make remote %q: %v", remoteName, err)
 	}
-	fstest.CheckListing(fileRemote, []fstest.Item{})
+	fstest.CheckListing(t, fileRemote, []fstest.Item{})
 	_, ok := fileRemote.(*fs.Limited)
 	if ok {
 		t.Errorf("%v is is a fs.Limited", fileRemote)
@@ -418,12 +414,12 @@ func TestObjectRemove(t *testing.T) {
 	if err != nil {
 		t.Fatal("Remove error", err)
 	}
-	fstest.CheckListing(remote, []fstest.Item{file2})
+	fstest.CheckListing(t, remote, []fstest.Item{file2})
 }
 
 func TestObjectPurge(t *testing.T) {
 	skipIfNotOk(t)
-	fstest.TestPurge(remote)
+	fstest.TestPurge(t, remote)
 	err := fs.Purge(remote)
 	if err == nil {
 		t.Fatal("Expecting error after on second purge")

notes.txt

@@ -3,14 +3,13 @@ Change lsd command so it doesn't show -1
   * Make test?
 
 Put the TestRemote names into the Fs description
-Make rclonetest use the TestRemote name automatically
+Make test_all.sh use the TestRemote name automatically
-Put rclonetest back into rclone as tests
-  * defaults to using local remote
-  * but could pass another in with a flag
 
 Run errcheck and go vet in the make file
 .. Also race detector?
 
+Get rid of Storable?
+
 Write developer manual
 
 Todo

rclone.go

@@ -96,7 +96,7 @@ var Commands = []Command{
 		Help: `
         List all the objects in the the path with size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.List(fdst)
+			err := fs.List(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}
@@ -110,7 +110,7 @@ var Commands = []Command{
 		Help: `
         List all directories/containers/buckets in the the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListDir(fdst)
+			err := fs.ListDir(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to listdir: %v", err)
 			}
@@ -124,7 +124,7 @@ var Commands = []Command{
 		Help: `
         List all the objects in the the path with modification time, size and path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.ListLong(fdst)
+			err := fs.ListLong(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list long: %v", err)
 			}
@@ -138,7 +138,7 @@ var Commands = []Command{
 		Help: `
         Produces an md5sum file for all the objects in the path.`,
 		Run: func(fdst, fsrc fs.Fs) {
-			err := fs.Md5sum(fdst)
+			err := fs.Md5sum(fdst, os.Stdout)
 			if err != nil {
 				log.Fatalf("Failed to list: %v", err)
 			}

rclonetest/rclonetest.go

@@ -1,290 +0,0 @@
-// Test rclone by doing real transactions to a storage provider to and
-// from the local disk
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path"
-	"time"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest"
-	"github.com/ogier/pflag"
-
-	// Active file systems
-	_ "github.com/ncw/rclone/drive"
-	_ "github.com/ncw/rclone/dropbox"
-	_ "github.com/ncw/rclone/googlecloudstorage"
-	_ "github.com/ncw/rclone/local"
-	_ "github.com/ncw/rclone/s3"
-	_ "github.com/ncw/rclone/swift"
-)
-
-// Globals
-var (
-	localName, remoteName string
-	version               = pflag.BoolP("version", "V", false, "Print the version number")
-	subDir                = pflag.BoolP("subdir", "S", false, "Test with a sub directory")
-)
-
-// Write a file
-func WriteFile(filePath, content string, t time.Time) {
-	// FIXME make directories?
-	filePath = path.Join(localName, filePath)
-	dirPath := path.Dir(filePath)
-	err := os.MkdirAll(dirPath, 0770)
-	if err != nil {
-		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
-	}
-	err = ioutil.WriteFile(filePath, []byte(content), 0600)
-	if err != nil {
-		log.Fatalf("Failed to write file %q: %v", filePath, err)
-	}
-	err = os.Chtimes(filePath, t, t)
-	if err != nil {
-		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
-	}
-}
-
-var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
-var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
-var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
-
-func TestCopy(flocal, fremote fs.Fs) {
-	WriteFile("sub dir/hello world", "hello world", t1)
-
-	// Check dry run is working
-	log.Printf("Copy with --dry-run")
-	fs.Config.DryRun = true
-	err := fs.Sync(fremote, flocal, false)
-	fs.Config.DryRun = false
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	items := []fstest.Item{
-		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
-	}
-
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, []fstest.Item{})
-
-	// Now without dry run
-
-	log.Printf("Copy")
-	err = fs.Sync(fremote, flocal, false)
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// Now delete the local file and download it
-
-	err = os.Remove(localName + "/sub dir/hello world")
-	if err != nil {
-		log.Fatalf("Remove failed: %v", err)
-	}
-
-	fstest.CheckListing(flocal, []fstest.Item{})
-	fstest.CheckListing(fremote, items)
-
-	log.Printf("Copy - redownload")
-	err = fs.Sync(flocal, fremote, false)
-	if err != nil {
-		log.Fatalf("Copy failed: %v", err)
-	}
-
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// Clean the directory
-	cleanTempDir()
-}
-
-func TestSync(flocal, fremote fs.Fs) {
-	WriteFile("empty space", "", t1)
-
-	log.Printf("Sync after changing file modtime only")
-	err := os.Chtimes(localName+"/empty space", t2, t2)
-	if err != nil {
-		log.Fatalf("Chtimes failed: %v", err)
-	}
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after adding a file")
-	WriteFile("potato", "------------------------------------------------------------", t3)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after changing a file's size only")
-	WriteFile("potato", "smaller but same date", t3)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after changing a file's contents, modtime but not length")
-	WriteFile("potato", "SMALLER BUT SAME DATE", t2)
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	items = []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-
-	// ------------------------------------------------------------
-
-	log.Printf("Sync after removing a file and adding a file --dry-run")
-	WriteFile("potato2", "------------------------------------------------------------", t1)
-	err = os.Remove(localName + "/potato")
-	if err != nil {
-		log.Fatalf("Remove failed: %v", err)
-	}
-	fs.Config.DryRun = true
-	err = fs.Sync(fremote, flocal, true)
-	fs.Config.DryRun = false
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-
-	before := []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
-	}
-	items = []fstest.Item{
-		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
-		{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, before)
-
-	log.Printf("Sync after removing a file and adding a file")
-	err = fs.Sync(fremote, flocal, true)
-	if err != nil {
-		log.Fatalf("Sync failed: %v", err)
-	}
-	fstest.CheckListing(flocal, items)
-	fstest.CheckListing(fremote, items)
-}
-
-func TestLs(flocal, fremote fs.Fs) {
-	// Underlying List has been tested above, so we just make sure it runs
-	err := fs.List(fremote)
-	if err != nil {
-		log.Fatalf("List failed: %v", err)
-	}
-}
-
-func TestLsd(flocal, fremote fs.Fs) {
-}
-
-func TestCheck(flocal, fremote fs.Fs) {
-}
-
-func syntaxError() {
-	fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s.
-
-Syntax: [options] remote:
-
-Need a remote: as argument.  This will create a random container or
-directory under it and perform tests on it, deleting it at the end.
-
-Options:
-
-`, fs.Version)
-	pflag.PrintDefaults()
-}
-
-// Clean the temporary directory
-func cleanTempDir() {
-	log.Printf("Cleaning temporary directory: %q", localName)
-	err := os.RemoveAll(localName)
-	if err != nil {
-		log.Printf("Failed to remove %q: %v", localName, err)
-	}
-}
-
-func main() {
-	pflag.Usage = syntaxError
-	pflag.Parse()
-	if *version {
-		fmt.Printf("rclonetest %s\n", fs.Version)
-		os.Exit(0)
-	}
-	fs.LoadConfig()
-	args := pflag.Args()
-
-	if len(args) != 1 {
-		syntaxError()
-		os.Exit(1)
-	}
-
-	fremote, finalise, err := fstest.RandomRemote(args[0], *subDir)
-	if err != nil {
-		log.Fatalf("Failed to open remote %q: %v", args[0], err)
-	}
-	log.Printf("Testing with remote %v", fremote)
-
-	localName, err = ioutil.TempDir("", "rclone")
-	if err != nil {
-		log.Fatalf("Failed to create temp dir: %v", err)
-	}
-	log.Printf("Testing with local %q", localName)
-	flocal, err := fs.NewFs(localName)
-	if err != nil {
-		log.Fatalf("Failed to make %q: %v", remoteName, err)
-	}
-
-	fs.CalculateModifyWindow(fremote, flocal)
-
-	fstest.TestMkdir(fremote)
-	TestCopy(flocal, fremote)
-	TestSync(flocal, fremote)
-	TestLs(flocal, fremote)
-	TestLsd(flocal, fremote)
-	TestCheck(flocal, fremote)
-	//TestRmdir(flocal, fremote)
-
-	finalise()
-
-	cleanTempDir()
-	log.Printf("Tests OK")
-}

rclonetest/test.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-go install
-
-REMOTES="
-TestSwift:
-TestS3:
-TestDrive:
-TestGoogleCloudStorage:
-TestDropbox:
-/tmp/z
-"
-
-function test_remote {
-    args=$@
-    rclonetest $args || {
-        echo "*** rclonetest $args FAILED ***"
-        exit 1
-    }
-}
-
-for remote in $REMOTES; do
-    test_remote $remote
-    test_remote --subdir $remote
-done