filter: add --ignore-case flag - fixes #502

The --ignore-case flag causes the filtering of file names to be case
insensitive.  The flag name comes from GNU tar.

docs/content/filtering.md

@@ -83,6 +83,18 @@ Special characters can be escaped with a `\` before them.
     \\.jpg       - matches "\.jpg"
     \[one\].jpg  - matches "[one].jpg"
 
+Patterns are case sensitive unless the `--ignore-case` flag is used.
+
+Without `--ignore-case` (default)
+
+    potato - matches "potato"
+           - doesn't match "POTATO"
+
+With `--ignore-case`
+
+    potato - matches "potato"
+           - matches "POTATO"
+
 Note also that rclone filter globs can only be used in one of the
 filter command line flags, not in the specification of the remote, so
 `rclone copy "remote:dir*.jpg" /path/to/dir` won't work - what is
@@ -431,6 +443,15 @@ This dumps the defined filters to the output as regular expressions.
 
 Useful for debugging.
 
+### `--ignore-case` - make searches case insensitive ###
+
+Normally filter patterns are case sensitive.  If this flag is supplied
+then filter patterns become case insensitive.
+
+Normally a `--include "file.txt"` will not match a file called
+`FILE.txt`.  However if you use the `--ignore-case` flag then
+`--include "file.txt"` this will match a file called `FILE.txt`.
+
 ## Quoting shell metacharacters ##
 
 The examples above may not work verbatim in your shell as they have

fs/filter/filter.go

@@ -90,6 +90,7 @@ type Opt struct {
 	MaxAge         fs.Duration
 	MinSize        fs.SizeSuffix
 	MaxSize        fs.SizeSuffix
+	IgnoreCase     bool
 }
 
 // DefaultOpt is the default config for the filter
@@ -226,7 +227,7 @@ func (f *Filter) addDirGlobs(Include bool, glob string) error {
 		if dirGlob == "/" {
 			continue
 		}
-		dirRe, err := globToRegexp(dirGlob)
+		dirRe, err := globToRegexp(dirGlob, f.Opt.IgnoreCase)
 		if err != nil {
 			return err
 		}
@@ -242,7 +243,7 @@ func (f *Filter) Add(Include bool, glob string) error {
 	if strings.Contains(glob, "**") {
 		isDirRule, isFileRule = true, true
 	}
-	re, err := globToRegexp(glob)
+	re, err := globToRegexp(glob, f.Opt.IgnoreCase)
 	if err != nil {
 		return err
 	}

fs/filter/filter_test.go

@@ -351,6 +351,7 @@ func TestNewFilterMatches(t *testing.T) {
 		{"cleared", 100, 0, false},
 		{"file1.jpg", 100, 0, false},
 		{"file2.png", 100, 0, true},
+		{"FILE2.png", 100, 0, false},
 		{"afile2.png", 100, 0, false},
 		{"file3.jpg", 101, 0, true},
 		{"file4.png", 101, 0, false},
@@ -370,6 +371,7 @@ func TestNewFilterMatches(t *testing.T) {
 		{"sausage2/sub", false},
 		{"sausage2/sub/dir", false},
 		{"sausage3", true},
+		{"SAUSAGE3", false},
 		{"sausage3/sub", true},
 		{"sausage3/sub/dir", true},
 		{"sausage4", false},
@@ -378,6 +380,28 @@ func TestNewFilterMatches(t *testing.T) {
 	assert.False(t, f.InActive())
 }
 
+func TestNewFilterMatchesIgnoreCase(t *testing.T) {
+	f, err := NewFilter(nil)
+	require.NoError(t, err)
+	f.Opt.IgnoreCase = true
+	add := func(s string) {
+		err := f.AddRule(s)
+		require.NoError(t, err)
+	}
+	add("+ /file2.png")
+	add("+ /sausage3**")
+	add("- *")
+	testInclude(t, f, []includeTest{
+		{"file2.png", 100, 0, true},
+		{"FILE2.png", 100, 0, true},
+	})
+	testDirInclude(t, f, []includeDirTest{
+		{"sausage3", true},
+		{"SAUSAGE3", true},
+	})
+	assert.False(t, f.InActive())
+}
+
 func TestFilterAddDirRuleOrFileRule(t *testing.T) {
 	for _, test := range []struct {
 		included bool
@@ -473,37 +497,45 @@ func TestFilterMatchesFromDocs(t *testing.T) {
 		glob       string
 		included   bool
 		file       string
+		ignoreCase bool
 	}{
-		{"file.jpg", true, "file.jpg"},
-		{"file.jpg", true, "directory/file.jpg"},
-		{"file.jpg", false, "afile.jpg"},
-		{"file.jpg", false, "directory/afile.jpg"},
-		{"/file.jpg", true, "file.jpg"},
-		{"/file.jpg", false, "afile.jpg"},
-		{"/file.jpg", false, "directory/file.jpg"},
-		{"*.jpg", true, "file.jpg"},
-		{"*.jpg", true, "directory/file.jpg"},
-		{"*.jpg", false, "file.jpg/anotherfile.png"},
-		{"dir/**", true, "dir/file.jpg"},
-		{"dir/**", true, "dir/dir1/dir2/file.jpg"},
-		{"dir/**", false, "directory/file.jpg"},
-		{"dir/**", false, "adir/file.jpg"},
-		{"l?ss", true, "less"},
-		{"l?ss", true, "lass"},
-		{"l?ss", false, "floss"},
-		{"h[ae]llo", true, "hello"},
-		{"h[ae]llo", true, "hallo"},
-		{"h[ae]llo", false, "hullo"},
-		{"{one,two}_potato", true, "one_potato"},
-		{"{one,two}_potato", true, "two_potato"},
-		{"{one,two}_potato", false, "three_potato"},
-		{"{one,two}_potato", false, "_potato"},
-		{"\\*.jpg", true, "*.jpg"},
-		{"\\\\.jpg", true, "\\.jpg"},
-		{"\\[one\\].jpg", true, "[one].jpg"},
+		{"file.jpg", true, "file.jpg", false},
+		{"file.jpg", true, "directory/file.jpg", false},
+		{"file.jpg", false, "afile.jpg", false},
+		{"file.jpg", false, "directory/afile.jpg", false},
+		{"/file.jpg", true, "file.jpg", false},
+		{"/file.jpg", false, "afile.jpg", false},
+		{"/file.jpg", false, "directory/file.jpg", false},
+		{"*.jpg", true, "file.jpg", false},
+		{"*.jpg", true, "directory/file.jpg", false},
+		{"*.jpg", false, "file.jpg/anotherfile.png", false},
+		{"dir/**", true, "dir/file.jpg", false},
+		{"dir/**", true, "dir/dir1/dir2/file.jpg", false},
+		{"dir/**", false, "directory/file.jpg", false},
+		{"dir/**", false, "adir/file.jpg", false},
+		{"l?ss", true, "less", false},
+		{"l?ss", true, "lass", false},
+		{"l?ss", false, "floss", false},
+		{"h[ae]llo", true, "hello", false},
+		{"h[ae]llo", true, "hallo", false},
+		{"h[ae]llo", false, "hullo", false},
+		{"{one,two}_potato", true, "one_potato", false},
+		{"{one,two}_potato", true, "two_potato", false},
+		{"{one,two}_potato", false, "three_potato", false},
+		{"{one,two}_potato", false, "_potato", false},
+		{"\\*.jpg", true, "*.jpg", false},
+		{"\\\\.jpg", true, "\\.jpg", false},
+		{"\\[one\\].jpg", true, "[one].jpg", false},
+		{"potato", true, "potato", false},
+		{"potato", false, "POTATO", false},
+		{"potato", true, "potato", true},
+		{"potato", true, "POTATO", true},
 	} {
 		f, err := NewFilter(nil)
 		require.NoError(t, err)
+		if test.ignoreCase {
+			f.Opt.IgnoreCase = true
+		}
 		err = f.Add(true, test.glob)
 		require.NoError(t, err)
 		err = f.Add(false, "*")

fs/filter/filterflags/filterflags.go

@@ -29,5 +29,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y")
 	flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in k or suffix b|k|M|G")
 	flags.FVarP(flagSet, &Opt.MaxSize, "max-size", "", "Only transfer files smaller than this in k or suffix b|k|M|G")
+	flags.BoolVarP(flagSet, &Opt.IgnoreCase, "ignore-case", "", false, "Ignore case in filters (case insensitive)")
 	//cvsExclude     = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
 }

fs/filter/glob.go

@@ -13,8 +13,11 @@ import (
 // globToRegexp converts an rsync style glob to a regexp
 //
 // documented in filtering.md
-func globToRegexp(glob string) (*regexp.Regexp, error) {
+func globToRegexp(glob string, ignoreCase bool) (*regexp.Regexp, error) {
 	var re bytes.Buffer
+	if ignoreCase {
+		_, _ = re.WriteString("(?i)")
+	}
 	if strings.HasPrefix(glob, "/") {
 		glob = glob[1:]
 		_, _ = re.WriteRune('^')

fs/filter/glob_test.go

@@ -41,11 +41,16 @@ func TestGlobToRegexp(t *testing.T) {
 		{`a\*b`, `(^|/)a\*b$`, ``},
 		{`a\\b`, `(^|/)a\\b$`, ``},
 	} {
-		gotRe, err := globToRegexp(test.in)
+		for _, ignoreCase := range []bool{false, true} {
+			gotRe, err := globToRegexp(test.in, ignoreCase)
 			if test.error == "" {
+				prefix := ""
+				if ignoreCase {
+					prefix = "(?i)"
+				}
 				got := gotRe.String()
 				require.NoError(t, err, test.in)
-			assert.Equal(t, test.want, got, test.in)
+				assert.Equal(t, prefix+test.want, got, test.in)
 			} else {
 				require.Error(t, err, test.in)
 				assert.Contains(t, err.Error(), test.error, test.in)
@@ -53,6 +58,7 @@ func TestGlobToRegexp(t *testing.T) {
 			}
 		}
 	}
+}
 
 func TestGlobToDirGlobs(t *testing.T) {
 	for _, test := range []struct {
@@ -96,7 +102,7 @@ func TestGlobToDirGlobs(t *testing.T) {
 		{"/sausage3**", []string{`/sausage3**/`, "/"}},
 		{"/a/*.jpg", []string{`/a/`, "/"}},
 	} {
-		_, err := globToRegexp(test.in)
+		_, err := globToRegexp(test.in, false)
 		assert.NoError(t, err)
 		got := globToDirGlobs(test.in)
 		assert.Equal(t, test.want, got, test.in)