1
mirror of https://github.com/rclone/rclone synced 2024-12-24 15:43:45 +01:00
rclone/fs/march/march_test.go

507 lines
11 KiB
Go

// Internal tests for march
package march
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/mockdir"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Some times used in the tests
var (
t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
)
func TestMain(m *testing.M) {
fstest.TestMain(m)
}
type marchTester struct {
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
srcOnly fs.DirEntries
dstOnly fs.DirEntries
match fs.DirEntries
entryMutex sync.Mutex
errorMu sync.Mutex // Mutex covering the error variables
err error
noRetryErr error
fatalErr error
noTraverse bool
}
// DstOnly have an object which is in the destination only
func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) {
mt.entryMutex.Lock()
mt.dstOnly = append(mt.dstOnly, dst)
mt.entryMutex.Unlock()
switch dst.(type) {
case fs.Object:
return false
case fs.Directory:
return true
default:
panic("Bad object in DirEntries")
}
}
// SrcOnly have an object which is in the source only
func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) {
mt.entryMutex.Lock()
mt.srcOnly = append(mt.srcOnly, src)
mt.entryMutex.Unlock()
switch src.(type) {
case fs.Object:
return false
case fs.Directory:
return true
default:
panic("Bad object in DirEntries")
}
}
// Match is called when src and dst are present, so sync src to dst
func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
mt.entryMutex.Lock()
mt.match = append(mt.match, src)
mt.entryMutex.Unlock()
switch src.(type) {
case fs.Object:
return false
case fs.Directory:
// Do the same thing to the entire contents of the directory
_, ok := dst.(fs.Directory)
if ok {
return true
}
// FIXME src is dir, dst is file
err := errors.New("can't overwrite file with directory")
fs.Errorf(dst, "%v", err)
mt.processError(err)
default:
panic("Bad object in DirEntries")
}
return false
}
func (mt *marchTester) processError(err error) {
if err == nil {
return
}
mt.errorMu.Lock()
defer mt.errorMu.Unlock()
switch {
case fserrors.IsFatalError(err):
if !mt.aborting() {
fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err)
mt.cancel()
}
mt.fatalErr = err
case fserrors.IsNoRetryError(err):
mt.noRetryErr = err
default:
mt.err = err
}
}
func (mt *marchTester) currentError() error {
mt.errorMu.Lock()
defer mt.errorMu.Unlock()
if mt.fatalErr != nil {
return mt.fatalErr
}
if mt.err != nil {
return mt.err
}
return mt.noRetryErr
}
func (mt *marchTester) aborting() bool {
return mt.ctx.Err() != nil
}
func TestMarch(t *testing.T) {
for _, test := range []struct {
what string
fileSrcOnly []string
dirSrcOnly []string
fileDstOnly []string
dirDstOnly []string
fileMatch []string
dirMatch []string
}{
{
what: "source only",
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
dirSrcOnly: []string{"sub dir"},
},
{
what: "identical",
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
dirMatch: []string{"sub dir", "sub dir/sub sub dir"},
},
{
what: "typical sync",
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
dirSrcOnly: []string{"srcOnlyDir"},
fileMatch: []string{"match", "matchDir/match file"},
dirMatch: []string{"matchDir"},
fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"},
dirDstOnly: []string{"dstOnlyDir"},
},
} {
t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
var srcOnly []fstest.Item
var dstOnly []fstest.Item
var match []fstest.Item
ctx, cancel := context.WithCancel(context.Background())
for _, f := range test.fileSrcOnly {
srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))
}
for _, f := range test.fileDstOnly {
dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1))
}
for _, f := range test.fileMatch {
match = append(match, r.WriteBoth(ctx, f, "hello world", t1))
}
mt := &marchTester{
ctx: ctx,
cancel: cancel,
noTraverse: false,
}
m := &March{
Ctx: ctx,
Fdst: r.Fremote,
Fsrc: r.Flocal,
Dir: "",
NoTraverse: mt.noTraverse,
Callback: mt,
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
}
mt.processError(m.Run())
mt.cancel()
err := mt.currentError()
require.NoError(t, err)
precision := fs.GetModifyWindow(r.Fremote, r.Flocal)
fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly")
fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, precision, "dstOnly")
fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match")
})
}
}
func TestMarchNoTraverse(t *testing.T) {
for _, test := range []struct {
what string
fileSrcOnly []string
dirSrcOnly []string
fileMatch []string
dirMatch []string
}{
{
what: "source only",
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
dirSrcOnly: []string{"sub dir"},
},
{
what: "identical",
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
},
{
what: "typical sync",
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
fileMatch: []string{"match", "matchDir/match file"},
},
} {
t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
var srcOnly []fstest.Item
var match []fstest.Item
ctx, cancel := context.WithCancel(context.Background())
for _, f := range test.fileSrcOnly {
srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))
}
for _, f := range test.fileMatch {
match = append(match, r.WriteBoth(ctx, f, "hello world", t1))
}
mt := &marchTester{
ctx: ctx,
cancel: cancel,
noTraverse: true,
}
m := &March{
Ctx: ctx,
Fdst: r.Fremote,
Fsrc: r.Flocal,
Dir: "",
NoTraverse: mt.noTraverse,
Callback: mt,
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
}
mt.processError(m.Run())
mt.cancel()
err := mt.currentError()
require.NoError(t, err)
precision := fs.GetModifyWindow(r.Fremote, r.Flocal)
fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly")
fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match")
})
}
}
func TestNewMatchEntries(t *testing.T) {
var (
a = mockobject.Object("path/a")
A = mockobject.Object("path/A")
B = mockobject.Object("path/B")
c = mockobject.Object("path/c")
)
es := newMatchEntries(fs.DirEntries{a, A, B, c}, nil)
assert.Equal(t, es, matchEntries{
{name: "A", leaf: "A", entry: A},
{name: "B", leaf: "B", entry: B},
{name: "a", leaf: "a", entry: a},
{name: "c", leaf: "c", entry: c},
})
es = newMatchEntries(fs.DirEntries{a, A, B, c}, []matchTransformFn{strings.ToLower})
assert.Equal(t, es, matchEntries{
{name: "a", leaf: "A", entry: A},
{name: "a", leaf: "a", entry: a},
{name: "b", leaf: "B", entry: B},
{name: "c", leaf: "c", entry: c},
})
}
func TestMatchListings(t *testing.T) {
var (
a = mockobject.Object("a")
A = mockobject.Object("A")
b = mockobject.Object("b")
c = mockobject.Object("c")
d = mockobject.Object("d")
dirA = mockdir.New("A")
dirb = mockdir.New("b")
)
for _, test := range []struct {
what string
input fs.DirEntries // pairs of input src, dst
srcOnly fs.DirEntries
dstOnly fs.DirEntries
matches []matchPair // pairs of output
transforms []matchTransformFn
}{
{
what: "only src or dst",
input: fs.DirEntries{
a, nil,
b, nil,
c, nil,
d, nil,
},
srcOnly: fs.DirEntries{
a, b, c, d,
},
},
{
what: "typical sync #1",
input: fs.DirEntries{
a, nil,
b, b,
nil, c,
nil, d,
},
srcOnly: fs.DirEntries{
a,
},
dstOnly: fs.DirEntries{
c, d,
},
matches: []matchPair{
{b, b},
},
},
{
what: "typical sync #2",
input: fs.DirEntries{
a, a,
b, b,
nil, c,
d, d,
},
dstOnly: fs.DirEntries{
c,
},
matches: []matchPair{
{a, a},
{b, b},
{d, d},
},
},
{
what: "One duplicate",
input: fs.DirEntries{
A, A,
a, a,
a, nil,
b, b,
},
matches: []matchPair{
{A, A},
{a, a},
{b, b},
},
},
{
what: "Two duplicates",
input: fs.DirEntries{
a, a,
a, a,
a, nil,
},
matches: []matchPair{
{a, a},
},
},
{
what: "Case insensitive duplicate - no transform",
input: fs.DirEntries{
a, a,
A, A,
},
matches: []matchPair{
{A, A},
{a, a},
},
},
{
what: "Case insensitive duplicate - transform to lower case",
input: fs.DirEntries{
a, a,
A, A,
},
matches: []matchPair{
{A, A},
},
transforms: []matchTransformFn{strings.ToLower},
},
{
what: "File and directory are not duplicates - srcOnly",
input: fs.DirEntries{
dirA, nil,
A, nil,
},
srcOnly: fs.DirEntries{
dirA,
A,
},
},
{
what: "File and directory are not duplicates - matches",
input: fs.DirEntries{
dirA, dirA,
A, A,
},
matches: []matchPair{
{dirA, dirA},
{A, A},
},
},
{
what: "Sync with directory #1",
input: fs.DirEntries{
dirA, nil,
A, nil,
b, b,
nil, c,
nil, d,
},
srcOnly: fs.DirEntries{
dirA,
A,
},
dstOnly: fs.DirEntries{
c, d,
},
matches: []matchPair{
{b, b},
},
},
{
what: "Sync with 2 directories",
input: fs.DirEntries{
dirA, dirA,
A, nil,
nil, dirb,
nil, b,
},
srcOnly: fs.DirEntries{
A,
},
dstOnly: fs.DirEntries{
dirb,
b,
},
matches: []matchPair{
{dirA, dirA},
},
},
} {
t.Run(fmt.Sprintf("TestMatchListings-%s", test.what), func(t *testing.T) {
var srcList, dstList fs.DirEntries
for i := 0; i < len(test.input); i += 2 {
src, dst := test.input[i], test.input[i+1]
if src != nil {
srcList = append(srcList, src)
}
if dst != nil {
dstList = append(dstList, dst)
}
}
srcOnly, dstOnly, matches := matchListings(srcList, dstList, test.transforms)
assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")
assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ")
assert.Equal(t, test.matches, matches, test.what, "matches differ")
// now swap src and dst
dstOnly, srcOnly, matches = matchListings(dstList, srcList, test.transforms)
assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")
assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ")
assert.Equal(t, test.matches, matches, test.what, "matches differ")
})
}
}