Add rclone info internal command for testing out limits of remotes

cmd/all/all.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/delete"
 	_ "github.com/ncw/rclone/cmd/genautocomplete"
 	_ "github.com/ncw/rclone/cmd/gendocs"
+	_ "github.com/ncw/rclone/cmd/info"
 	_ "github.com/ncw/rclone/cmd/listremotes"
 	_ "github.com/ncw/rclone/cmd/ls"
 	_ "github.com/ncw/rclone/cmd/ls2"

cmd/info/all.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+exec rclone --check-normalization=true --check-control=true --check-length=true info \
+	/tmp/testInfo \
+	TestAmazonCloudDrive:testInfo \
+	TestB2:testInfo \
+	TestCryptDrive:testInfo \
+	TestCryptSwift:testInfo \
+	TestDrive:testInfo \
+	TestDropbox:testInfo \
+	TestGoogleCloudStorage:rclone-testinfo \
+	TestOneDrive:testInfo \
+	TestS3:rclone-testinfo \
+	TestSftp:testInfo \
+	TestSwift:testInfo \
+	TestYandex:testInfo \
+	TestFTP:testInfo
+
+#	TestHubic:testInfo \

cmd/info/info.go

@@ -0,0 +1,214 @@
+package info
+
+// FIXME once translations are implemented will need a no-escape
+// option for Put so we can make these tests work agaig
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	checkNormalization bool
+	checkControl       bool
+	checkLength        bool
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
+	commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.")
+	commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.")
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "info [remote:path]+",
+	Short: `Discovers file name limitations for paths.`,
+	Long: `rclone info discovers what filenames are possible to write to the
+paths passed in and how long they can be.  It can take some time.  It
+will write test files into the remote:path passed in.  It outputs a bit
+of go code for each one.
+`,
+	Hidden: true,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1E6, command, args)
+		for i := range args {
+			f := cmd.NewFsDst(args[i : i+1])
+			cmd.Run(false, false, command, func() error {
+				return readInfo(f)
+			})
+		}
+	},
+}
+
+type results struct {
+	f                    fs.Fs
+	mu                   sync.Mutex
+	charNeedsEscaping    map[rune]bool
+	maxFileLength        int
+	canWriteUnnormalized bool
+	canReadUnnormalized  bool
+	canReadRenormalized  bool
+}
+
+func newResults(f fs.Fs) *results {
+	return &results{
+		f:                 f,
+		charNeedsEscaping: make(map[rune]bool),
+	}
+}
+
+// Print the results to stdout
+func (r *results) Print() {
+	fmt.Printf("// %s\n", r.f.Name())
+	if checkControl {
+		escape := []string{}
+		for c, needsEscape := range r.charNeedsEscaping {
+			if needsEscape {
+				escape = append(escape, fmt.Sprintf("0x%02X", c))
+			}
+		}
+		sort.Strings(escape)
+		fmt.Printf("charNeedsEscaping = []byte{\n")
+		fmt.Printf("\t%s\n", strings.Join(escape, ", "))
+		fmt.Printf("}\n")
+	}
+	if checkLength {
+		fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
+	}
+	if checkNormalization {
+		fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
+		fmt.Printf("canReadUnnormalized   = %v\n", r.canReadUnnormalized)
+		fmt.Printf("canReadRenormalized   = %v\n", r.canReadRenormalized)
+	}
+}
+
+// writeFile writes a file with some random contents
+func (r *results) writeFile(path string) (fs.Object, error) {
+	contents := fstest.RandomString(50)
+	src := fs.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
+	return r.f.Put(bytes.NewBufferString(contents), src)
+}
+
+// check whether normalization is enforced and check whether it is
+// done on the files anyway
+func (r *results) checkUTF8Normalization() {
+	unnormalized := "Héroique"
+	normalized := "Héroique"
+	_, err := r.writeFile(unnormalized)
+	if err != nil {
+		r.canWriteUnnormalized = false
+		return
+	}
+	r.canWriteUnnormalized = true
+	_, err = r.f.NewObject(unnormalized)
+	if err == nil {
+		r.canReadUnnormalized = true
+	}
+	_, err = r.f.NewObject(normalized)
+	if err == nil {
+		r.canReadRenormalized = true
+	}
+}
+
+// check we can write file with the rune passed in
+func (r *results) checkChar(c rune) {
+	fs.Infof(r.f, "Writing file 0x%02X", c)
+	path := fmt.Sprintf("0x%02X-%c-", c, c)
+	_, err := r.writeFile(path)
+	escape := false
+	if err != nil {
+		fs.Infof(r.f, "Couldn't write file 0x%02X", c)
+	} else {
+		fs.Infof(r.f, "OK writing file 0x%02X", c)
+	}
+	r.mu.Lock()
+	r.charNeedsEscaping[c] = escape
+	r.mu.Unlock()
+}
+
+// check we can write a file with the control chars
+func (r *results) checkControls() {
+	fs.Infof(r.f, "Trying to create control character file names")
+	// Concurrency control
+	tokens := make(chan struct{}, fs.Config.Checkers)
+	for i := 0; i < fs.Config.Checkers; i++ {
+		tokens <- struct{}{}
+	}
+	var wg sync.WaitGroup
+	for i := rune(0); i < 128; i++ {
+		if i == 0 || i == '/' {
+			// We're not even going to check NULL or /
+			r.charNeedsEscaping[i] = true
+			continue
+		}
+		wg.Add(1)
+		c := i
+		go func() {
+			defer wg.Done()
+			token := <-tokens
+			r.checkChar(c)
+			tokens <- token
+		}()
+	}
+	wg.Wait()
+	fs.Infof(r.f, "Done trying to create control character file names")
+}
+
+// find the max file name size we can use
+func (r *results) findMaxLength() {
+	const maxLen = 16 * 1024
+	name := make([]byte, maxLen)
+	for i := range name {
+		name[i] = 'a'
+	}
+	// Find the first size of filename we can't write
+	i := sort.Search(len(name), func(i int) (fail bool) {
+		defer func() {
+			if err := recover(); err != nil {
+				fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
+				fail = true
+			}
+		}()
+
+		path := string(name[:i])
+		_, err := r.writeFile(path)
+		if err != nil {
+			fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
+			return true
+		}
+		fs.Infof(r.f, "Wrote file with name length %d", i)
+		return false
+	})
+	r.maxFileLength = i - 1
+	fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
+}
+
+func readInfo(f fs.Fs) error {
+	err := f.Mkdir("")
+	if err != nil {
+		return errors.Wrap(err, "couldn't mkdir")
+	}
+	r := newResults(f)
+	if checkControl {
+		r.checkControls()
+	}
+	if checkLength {
+		r.findMaxLength()
+	}
+	if checkNormalization {
+		r.checkUTF8Normalization()
+	}
+	r.Print()
+	return nil
+}