mirror of
https://github.com/rclone/rclone
synced 2024-12-03 13:14:37 +01:00
478 lines
11 KiB
Go
478 lines
11 KiB
Go
package fs
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestAgeSuffix(t *testing.T) {
|
|
for _, test := range []struct {
|
|
in string
|
|
want float64
|
|
err bool
|
|
}{
|
|
{"0", 0, false},
|
|
{"", 0, true},
|
|
{"1ms", float64(time.Millisecond), false},
|
|
{"1s", float64(time.Second), false},
|
|
{"1m", float64(time.Minute), false},
|
|
{"1h", float64(time.Hour), false},
|
|
{"1d", float64(time.Hour) * 24, false},
|
|
{"1w", float64(time.Hour) * 24 * 7, false},
|
|
{"1M", float64(time.Hour) * 24 * 30, false},
|
|
{"1y", float64(time.Hour) * 24 * 365, false},
|
|
{"1.5y", float64(time.Hour) * 24 * 365 * 1.5, false},
|
|
{"-1s", -float64(time.Second), false},
|
|
{"1.s", float64(time.Second), false},
|
|
{"1x", 0, true},
|
|
} {
|
|
duration, err := ParseDuration(test.in)
|
|
if test.err {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
assert.Equal(t, test.want, float64(duration))
|
|
}
|
|
}
|
|
|
|
func TestNewFilterDefault(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
assert.False(t, f.DeleteExcluded)
|
|
assert.Equal(t, int64(-1), f.MinSize)
|
|
assert.Equal(t, int64(-1), f.MaxSize)
|
|
assert.Len(t, f.fileRules.rules, 0)
|
|
assert.Len(t, f.dirRules.rules, 0)
|
|
assert.Nil(t, f.files)
|
|
assert.True(t, f.InActive())
|
|
}
|
|
|
|
// testFile creates a temp file with the contents
|
|
func testFile(t *testing.T, contents string) string {
|
|
out, err := ioutil.TempFile("", "filter_test")
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
err := out.Close()
|
|
require.NoError(t, err)
|
|
}()
|
|
_, err = out.Write([]byte(contents))
|
|
require.NoError(t, err)
|
|
s := out.Name()
|
|
return s
|
|
}
|
|
|
|
func TestNewFilterFull(t *testing.T) {
|
|
mins := int64(100 * 1024)
|
|
maxs := int64(1000 * 1024)
|
|
isFalse := false
|
|
isTrue := true
|
|
|
|
// Set up the input
|
|
deleteExcluded = &isTrue
|
|
filterRule = &[]string{"- filter1", "- filter1b"}
|
|
filterFrom = &[]string{testFile(t, "#comment\n+ filter2\n- filter3\n")}
|
|
excludeRule = &[]string{"exclude1"}
|
|
excludeFrom = &[]string{testFile(t, "#comment\nexclude2\nexclude3\n")}
|
|
includeRule = &[]string{"include1"}
|
|
includeFrom = &[]string{testFile(t, "#comment\ninclude2\ninclude3\n")}
|
|
filesFrom = &[]string{testFile(t, "#comment\nfiles1\nfiles2\n")}
|
|
minSize = SizeSuffix(mins)
|
|
maxSize = SizeSuffix(maxs)
|
|
|
|
rm := func(p string) {
|
|
err := os.Remove(p)
|
|
if err != nil {
|
|
t.Logf("error removing %q: %v", p, err)
|
|
}
|
|
}
|
|
// Reset the input
|
|
defer func() {
|
|
rm((*filterFrom)[0])
|
|
rm((*excludeFrom)[0])
|
|
rm((*includeFrom)[0])
|
|
rm((*filesFrom)[0])
|
|
minSize = -1
|
|
maxSize = -1
|
|
deleteExcluded = &isFalse
|
|
filterRule = nil
|
|
filterFrom = nil
|
|
excludeRule = nil
|
|
excludeFrom = nil
|
|
includeRule = nil
|
|
includeFrom = nil
|
|
filesFrom = nil
|
|
}()
|
|
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
assert.True(t, f.DeleteExcluded)
|
|
assert.Equal(t, f.MinSize, mins)
|
|
assert.Equal(t, f.MaxSize, maxs)
|
|
got := f.DumpFilters()
|
|
want := `--- File filter rules ---
|
|
+ (^|/)include1$
|
|
+ (^|/)include2$
|
|
+ (^|/)include3$
|
|
- (^|/)exclude1$
|
|
- (^|/)exclude2$
|
|
- (^|/)exclude3$
|
|
- (^|/)filter1$
|
|
- (^|/)filter1b$
|
|
+ (^|/)filter2$
|
|
- (^|/)filter3$
|
|
- ^.*$
|
|
--- Directory filter rules ---
|
|
+ ^.*$
|
|
- ^.*$`
|
|
assert.Equal(t, want, got)
|
|
assert.Len(t, f.files, 2)
|
|
for _, name := range []string{"files1", "files2"} {
|
|
_, ok := f.files[name]
|
|
if !ok {
|
|
t.Errorf("Didn't find file %q in f.files", name)
|
|
}
|
|
}
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
type includeTest struct {
|
|
in string
|
|
size int64
|
|
modTime int64
|
|
want bool
|
|
}
|
|
|
|
func testInclude(t *testing.T, f *Filter, tests []includeTest) {
|
|
for _, test := range tests {
|
|
got := f.Include(test.in, test.size, time.Unix(test.modTime, 0))
|
|
assert.Equal(t, test.want, got, test.in, test.size, test.modTime)
|
|
}
|
|
}
|
|
|
|
type includeDirTest struct {
|
|
in string
|
|
want bool
|
|
}
|
|
|
|
func testDirInclude(t *testing.T, f *Filter, tests []includeDirTest) {
|
|
for _, test := range tests {
|
|
got := f.IncludeDirectory(test.in)
|
|
assert.Equal(t, test.want, got, test.in)
|
|
}
|
|
}
|
|
|
|
func TestNewFilterIncludeFiles(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
err = f.AddFile("file1.jpg")
|
|
require.NoError(t, err)
|
|
err = f.AddFile("/file2.jpg")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, FilesMap{
|
|
"file1.jpg": {},
|
|
"file2.jpg": {},
|
|
}, f.files)
|
|
assert.Equal(t, FilesMap{}, f.dirs)
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 0, 0, true},
|
|
{"file2.jpg", 1, 0, true},
|
|
{"potato/file2.jpg", 2, 0, false},
|
|
{"file3.jpg", 3, 0, false},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterIncludeFilesDirs(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
for _, path := range []string{
|
|
"path/to/dir/file1.png",
|
|
"/path/to/dir/file2.png",
|
|
"/path/to/file3.png",
|
|
"/path/to/dir2/file4.png",
|
|
} {
|
|
err = f.AddFile(path)
|
|
require.NoError(t, err)
|
|
}
|
|
assert.Equal(t, FilesMap{
|
|
"path": {},
|
|
"path/to": {},
|
|
"path/to/dir": {},
|
|
"path/to/dir2": {},
|
|
}, f.dirs)
|
|
testDirInclude(t, f, []includeDirTest{
|
|
{"path", true},
|
|
{"path/to", true},
|
|
{"path/to/", true},
|
|
{"/path/to", true},
|
|
{"/path/to/", true},
|
|
{"path/to/dir", true},
|
|
{"path/to/dir2", true},
|
|
{"path/too", false},
|
|
{"path/three", false},
|
|
{"four", false},
|
|
})
|
|
}
|
|
|
|
func TestNewFilterMinSize(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
f.MinSize = 100
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 100, 0, true},
|
|
{"file2.jpg", 101, 0, true},
|
|
{"potato/file2.jpg", 99, 0, false},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterMaxSize(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
f.MaxSize = 100
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 100, 0, true},
|
|
{"file2.jpg", 101, 0, false},
|
|
{"potato/file2.jpg", 99, 0, true},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterMinAndMaxAge(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
f.ModTimeFrom = time.Unix(1440000002, 0)
|
|
f.ModTimeTo = time.Unix(1440000003, 0)
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 100, 1440000000, false},
|
|
{"file2.jpg", 101, 1440000001, false},
|
|
{"file3.jpg", 102, 1440000002, true},
|
|
{"potato/file1.jpg", 98, 1440000003, true},
|
|
{"potato/file2.jpg", 99, 1440000004, false},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterMinAge(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
f.ModTimeTo = time.Unix(1440000002, 0)
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 100, 1440000000, true},
|
|
{"file2.jpg", 101, 1440000001, true},
|
|
{"file3.jpg", 102, 1440000002, true},
|
|
{"potato/file1.jpg", 98, 1440000003, false},
|
|
{"potato/file2.jpg", 99, 1440000004, false},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterMaxAge(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
f.ModTimeFrom = time.Unix(1440000002, 0)
|
|
testInclude(t, f, []includeTest{
|
|
{"file1.jpg", 100, 1440000000, false},
|
|
{"file2.jpg", 101, 1440000001, false},
|
|
{"file3.jpg", 102, 1440000002, true},
|
|
{"potato/file1.jpg", 98, 1440000003, true},
|
|
{"potato/file2.jpg", 99, 1440000004, true},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestNewFilterMatches(t *testing.T) {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
add := func(s string) {
|
|
err := f.AddRule(s)
|
|
require.NoError(t, err)
|
|
}
|
|
add("+ cleared")
|
|
add("!")
|
|
add("- /file1.jpg")
|
|
add("+ /file2.png")
|
|
add("+ /*.jpg")
|
|
add("- /*.png")
|
|
add("- /potato")
|
|
add("+ /sausage1")
|
|
add("+ /sausage2*")
|
|
add("+ /sausage3**")
|
|
add("+ /a/*.jpg")
|
|
add("- *")
|
|
testInclude(t, f, []includeTest{
|
|
{"cleared", 100, 0, false},
|
|
{"file1.jpg", 100, 0, false},
|
|
{"file2.png", 100, 0, true},
|
|
{"afile2.png", 100, 0, false},
|
|
{"file3.jpg", 101, 0, true},
|
|
{"file4.png", 101, 0, false},
|
|
{"potato", 101, 0, false},
|
|
{"sausage1", 101, 0, true},
|
|
{"sausage1/potato", 101, 0, false},
|
|
{"sausage2potato", 101, 0, true},
|
|
{"sausage2/potato", 101, 0, false},
|
|
{"sausage3/potato", 101, 0, true},
|
|
{"a/one.jpg", 101, 0, true},
|
|
{"a/one.png", 101, 0, false},
|
|
{"unicorn", 99, 0, false},
|
|
})
|
|
testDirInclude(t, f, []includeDirTest{
|
|
{"sausage1", false},
|
|
{"sausage2", false},
|
|
{"sausage2/sub", false},
|
|
{"sausage2/sub/dir", false},
|
|
{"sausage3", true},
|
|
{"sausage3/sub", true},
|
|
{"sausage3/sub/dir", true},
|
|
{"sausage4", false},
|
|
{"a", true},
|
|
})
|
|
assert.False(t, f.InActive())
|
|
}
|
|
|
|
func TestFilterAddDirRuleOrFileRule(t *testing.T) {
|
|
for _, test := range []struct {
|
|
included bool
|
|
glob string
|
|
want string
|
|
}{
|
|
{
|
|
false,
|
|
"potato",
|
|
`--- File filter rules ---
|
|
- (^|/)potato$
|
|
--- Directory filter rules ---`,
|
|
},
|
|
{
|
|
true,
|
|
"potato",
|
|
`--- File filter rules ---
|
|
+ (^|/)potato$
|
|
--- Directory filter rules ---
|
|
+ ^.*$`,
|
|
},
|
|
{
|
|
false,
|
|
"*",
|
|
`--- File filter rules ---
|
|
- (^|/)[^/]*$
|
|
--- Directory filter rules ---
|
|
- ^.*$`,
|
|
},
|
|
{
|
|
true,
|
|
"*",
|
|
`--- File filter rules ---
|
|
+ (^|/)[^/]*$
|
|
--- Directory filter rules ---
|
|
+ ^.*$`,
|
|
},
|
|
{
|
|
false,
|
|
".*{,/**}",
|
|
`--- File filter rules ---
|
|
- (^|/)\.[^/]*(|/.*)$
|
|
--- Directory filter rules ---
|
|
- (^|/)\.[^/]*(|/.*)$`,
|
|
},
|
|
{
|
|
true,
|
|
"a/b/c/d",
|
|
`--- File filter rules ---
|
|
+ (^|/)a/b/c/d$
|
|
--- Directory filter rules ---
|
|
+ (^|/)a/b/c/$
|
|
+ (^|/)a/b/$
|
|
+ (^|/)a/$`,
|
|
},
|
|
} {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
err = f.Add(test.included, test.glob)
|
|
require.NoError(t, err)
|
|
got := f.DumpFilters()
|
|
assert.Equal(t, test.want, got, fmt.Sprintf("Add(%v, %q)", test.included, test.glob))
|
|
}
|
|
}
|
|
|
|
func TestFilterForEachLine(t *testing.T) {
|
|
file := testFile(t, `; comment
|
|
one
|
|
# another comment
|
|
|
|
|
|
two
|
|
# indented comment
|
|
three
|
|
four
|
|
five
|
|
six `)
|
|
defer func() {
|
|
err := os.Remove(file)
|
|
require.NoError(t, err)
|
|
}()
|
|
lines := []string{}
|
|
err := forEachLine(file, func(s string) error {
|
|
lines = append(lines, s)
|
|
return nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "one,two,three,four,five,six", strings.Join(lines, ","))
|
|
}
|
|
|
|
func TestFilterMatchesFromDocs(t *testing.T) {
|
|
for _, test := range []struct {
|
|
glob string
|
|
included bool
|
|
file string
|
|
}{
|
|
{"file.jpg", true, "file.jpg"},
|
|
{"file.jpg", true, "directory/file.jpg"},
|
|
{"file.jpg", false, "afile.jpg"},
|
|
{"file.jpg", false, "directory/afile.jpg"},
|
|
{"/file.jpg", true, "file.jpg"},
|
|
{"/file.jpg", false, "afile.jpg"},
|
|
{"/file.jpg", false, "directory/file.jpg"},
|
|
{"*.jpg", true, "file.jpg"},
|
|
{"*.jpg", true, "directory/file.jpg"},
|
|
{"*.jpg", false, "file.jpg/anotherfile.png"},
|
|
{"dir/**", true, "dir/file.jpg"},
|
|
{"dir/**", true, "dir/dir1/dir2/file.jpg"},
|
|
{"dir/**", false, "directory/file.jpg"},
|
|
{"dir/**", false, "adir/file.jpg"},
|
|
{"l?ss", true, "less"},
|
|
{"l?ss", true, "lass"},
|
|
{"l?ss", false, "floss"},
|
|
{"h[ae]llo", true, "hello"},
|
|
{"h[ae]llo", true, "hallo"},
|
|
{"h[ae]llo", false, "hullo"},
|
|
{"{one,two}_potato", true, "one_potato"},
|
|
{"{one,two}_potato", true, "two_potato"},
|
|
{"{one,two}_potato", false, "three_potato"},
|
|
{"{one,two}_potato", false, "_potato"},
|
|
{"\\*.jpg", true, "*.jpg"},
|
|
{"\\\\.jpg", true, "\\.jpg"},
|
|
{"\\[one\\].jpg", true, "[one].jpg"},
|
|
} {
|
|
f, err := NewFilter()
|
|
require.NoError(t, err)
|
|
err = f.Add(true, test.glob)
|
|
require.NoError(t, err)
|
|
err = f.Add(false, "*")
|
|
require.NoError(t, err)
|
|
included := f.Include(test.file, 0, time.Unix(0, 0))
|
|
if included != test.included {
|
|
t.Errorf("%q match %q: want %v got %v", test.glob, test.file, test.included, included)
|
|
}
|
|
}
|
|
}
|