Remove flattening and replace with {off, standard} name encryption

crypt/cipher.go

@@ -6,6 +6,7 @@ import (
 	gocipher "crypto/cipher"
 	"crypto/rand"
 	"encoding/base32"
+	"fmt"
 	"io"
 	"strings"
 	"sync"
@@ -30,6 +31,7 @@ const (
 	blockHeaderSize     = secretbox.Overhead
 	blockDataSize       = 64 * 1024
 	blockSize           = blockHeaderSize + blockDataSize
+	encryptedSuffix     = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
 )
 
 // Errors returned by cipher
@@ -43,10 +45,8 @@ var (
 	ErrorEncryptedBadMagic       = errors.New("not an encrypted file - bad magic string")
 	ErrorEncryptedBadBlock       = errors.New("failed to authenticate decrypted block - bad password?")
 	ErrorBadBase32Encoding       = errors.New("bad base32 filename encoding")
-	ErrorBadSpreadNotSingleChar  = errors.New("bad unspread - not single character")
-	ErrorBadSpreadResultTooShort = errors.New("bad unspread - result too short")
-	ErrorBadSpreadDidntMatch     = errors.New("bad unspread - directory prefix didn't match")
 	ErrorFileClosed              = errors.New("file already closed")
+	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
 	defaultSalt                  = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
 )
 
@@ -57,10 +57,14 @@ var (
 
 // Cipher is used to swap out the encryption implementations
 type Cipher interface {
-	// EncryptName encrypts a file path
-	EncryptName(string) string
-	// DecryptName decrypts a file path, returns error if decrypt was invalid
-	DecryptName(string) (string, error)
+	// EncryptFileName encrypts a file path
+	EncryptFileName(string) string
+	// DecryptFileName decrypts a file path, returns error if decrypt was invalid
+	DecryptFileName(string) (string, error)
+	// EncryptDirName encrypts a directory path
+	EncryptDirName(string) string
+	// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
+	DecryptDirName(string) (string, error)
 	// EncryptData
 	EncryptData(io.Reader) (io.Reader, error)
 	// DecryptData
@@ -71,20 +75,56 @@ type Cipher interface {
 	DecryptedSize(int64) (int64, error)
 }
 
+// NameEncryptionMode is the type of file name encryption in use
+type NameEncryptionMode int
+
+// NameEncryptionMode levels
+const (
+	NameEncryptionOff NameEncryptionMode = iota
+	NameEncryptionStandard
+)
+
+// NewNameEncryptionMode turns a string into a NameEncryptionMode
+func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
+	s = strings.ToLower(s)
+	switch s {
+	case "off":
+		mode = NameEncryptionOff
+	case "standard":
+		mode = NameEncryptionStandard
+	default:
+		err = errors.Errorf("Unknown file name encryption mode %q", s)
+	}
+	return mode, err
+}
+
+// String turns mode into a human readable string
+func (mode NameEncryptionMode) String() (out string) {
+	switch mode {
+	case NameEncryptionOff:
+		out = "off"
+	case NameEncryptionStandard:
+		out = "standard"
+	default:
+		out = fmt.Sprintf("Unknown mode #%d", mode)
+	}
+	return out
+}
+
 type cipher struct {
 	dataKey    [32]byte                  // Key for secretbox
 	nameKey    [32]byte                  // 16,24 or 32 bytes
 	nameTweak  [nameCipherBlockSize]byte // used to tweak the name crypto
 	block      gocipher.Block
-	flatten    int       // set flattening level - 0 is off
+	mode       NameEncryptionMode
 	buffers    sync.Pool // encrypt/decrypt buffers
 	cryptoRand io.Reader // read crypto random numbers from here
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(flatten int, password, salt string) (*cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string) (*cipher, error) {
 	c := &cipher{
-		flatten:    flatten,
+		mode:       mode,
 		cryptoRand: rand.Reader,
 	}
 	c.buffers.New = func() interface{} {
@@ -231,50 +271,8 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 	return string(plaintext), err
 }
 
-// spread a name over the given number of directory levels
-//
-// if in isn't long enough dirs will be reduces
-func spreadName(dirs int, in string) string {
-	if dirs > len(in) {
-		dirs = len(in)
-	}
-	prefix := ""
-	for i := 0; i < dirs; i++ {
-		prefix += string(in[i]) + "/"
-	}
-	return prefix + in
-}
-
-// reverse spreadName, returning an error if not in spread format
-//
-// This decodes any level of spreading
-func unspreadName(in string) (string, error) {
-	in = strings.ToLower(in)
-	segments := strings.Split(in, "/")
-	if len(segments) == 0 {
-		return in, nil
-	}
-	out := segments[len(segments)-1]
-	segments = segments[:len(segments)-1]
-	for i, s := range segments {
-		if len(s) != 1 {
-			return "", ErrorBadSpreadNotSingleChar
-		}
-		if i >= len(out) {
-			return "", ErrorBadSpreadResultTooShort
-		}
-		if s[0] != out[i] {
-			return "", ErrorBadSpreadDidntMatch
-		}
-	}
-	return out, nil
-}
-
-// EncryptName encrypts a file path
-func (c *cipher) EncryptName(in string) string {
-	if c.flatten > 0 {
-		return spreadName(c.flatten, c.encryptSegment(in))
-	}
+// encryptFileName encrypts a file path
+func (c *cipher) encryptFileName(in string) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		segments[i] = c.encryptSegment(segments[i])
@@ -282,15 +280,24 @@ func (c *cipher) EncryptName(in string) string {
 	return strings.Join(segments, "/")
 }
 
-// DecryptName decrypts a file path
-func (c *cipher) DecryptName(in string) (string, error) {
-	if c.flatten > 0 {
-		unspread, err := unspreadName(in)
-		if err != nil {
-			return "", err
-		}
-		return c.decryptSegment(unspread)
+// EncryptFileName encrypts a file path
+func (c *cipher) EncryptFileName(in string) string {
+	if c.mode == NameEncryptionOff {
+		return in + encryptedSuffix
 	}
+	return c.encryptFileName(in)
+}
+
+// EncryptDirName encrypts a directory path
+func (c *cipher) EncryptDirName(in string) string {
+	if c.mode == NameEncryptionOff {
+		return in
+	}
+	return c.encryptFileName(in)
+}
+
+// decryptFileName decrypts a file path
+func (c *cipher) decryptFileName(in string) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -302,6 +309,26 @@ func (c *cipher) DecryptName(in string) (string, error) {
 	return strings.Join(segments, "/"), nil
 }
 
+// DecryptFileName decrypts a file path
+func (c *cipher) DecryptFileName(in string) (string, error) {
+	if c.mode == NameEncryptionOff {
+		remainingLength := len(in) - len(encryptedSuffix)
+		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
+			return in[:remainingLength], nil
+		}
+		return "", ErrorNotAnEncryptedFile
+	}
+	return c.decryptFileName(in)
+}
+
+// DecryptDirName decrypts a directory path
+func (c *cipher) DecryptDirName(in string) (string, error) {
+	if c.mode == NameEncryptionOff {
+		return in, nil
+	}
+	return c.decryptFileName(in)
+}
+
 // nonce is an NACL secretbox nonce
 type nonce [fileNonceSize]byte
 

crypt/cipher_test.go

@@ -15,6 +15,32 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestNewNameEncryptionMode(t *testing.T) {
+	for _, test := range []struct {
+		in          string
+		expected    NameEncryptionMode
+		expectedErr string
+	}{
+		{"off", NameEncryptionOff, ""},
+		{"standard", NameEncryptionStandard, ""},
+		{"potato", NameEncryptionMode(0), "Unknown file name encryption mode \"potato\""},
+	} {
+		actual, actualErr := NewNameEncryptionMode(test.in)
+		assert.Equal(t, actual, test.expected)
+		if test.expectedErr == "" {
+			assert.NoError(t, actualErr)
+		} else {
+			assert.Error(t, actualErr, test.expectedErr)
+		}
+	}
+}
+
+func TestNewNameEncryptionModeString(t *testing.T) {
+	assert.Equal(t, NameEncryptionOff.String(), "off")
+	assert.Equal(t, NameEncryptionStandard.String(), "standard")
+	assert.Equal(t, NameEncryptionMode(2).String(), "Unknown mode #2")
+}
+
 func TestValidString(t *testing.T) {
 	for _, test := range []struct {
 		in       string
@@ -129,7 +155,7 @@ func TestDecodeFileName(t *testing.T) {
 }
 
 func TestEncryptSegment(t *testing.T) {
-	c, _ := newCipher(0, "", "")
+	c, _ := newCipher(NameEncryptionStandard, "", "")
 	for _, test := range []struct {
 		in       string
 		expected string
@@ -166,7 +192,7 @@ func TestEncryptSegment(t *testing.T) {
 
 func TestDecryptSegment(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
-	c, _ := newCipher(0, "", "")
+	c, _ := newCipher(NameEncryptionStandard, "", "")
 	for _, test := range []struct {
 		in          string
 		expectedErr error
@@ -184,87 +210,78 @@ func TestDecryptSegment(t *testing.T) {
 	}
 }
 
-func TestSpreadName(t *testing.T) {
-	for _, test := range []struct {
-		n        int
-		in       string
-		expected string
-	}{
-		{3, "", ""},
-		{0, "abcdefg", "abcdefg"},
-		{1, "abcdefg", "a/abcdefg"},
-		{2, "abcdefg", "a/b/abcdefg"},
-		{3, "abcdefg", "a/b/c/abcdefg"},
-		{4, "abcdefg", "a/b/c/d/abcdefg"},
-		{4, "abcd", "a/b/c/d/abcd"},
-		{4, "abc", "a/b/c/abc"},
-		{4, "ab", "a/b/ab"},
-		{4, "a", "a/a"},
-	} {
-		actual := spreadName(test.n, test.in)
-		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d,%q", test.n, test.in))
-		recovered, err := unspreadName(test.expected)
-		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
-		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
-	}
+func TestEncryptFileName(t *testing.T) {
+	// First standard mode
+	c, _ := newCipher(NameEncryptionStandard, "", "")
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
+	// Now off mode
+	c, _ = newCipher(NameEncryptionOff, "", "")
+	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 }
 
-func TestUnspreadName(t *testing.T) {
-	// We've tested the forwards above, now concentrate on the errors
+func TestDecryptFileName(t *testing.T) {
 	for _, test := range []struct {
-		in          string
-		expectedErr error
-	}{
-		{"aa/bc", ErrorBadSpreadNotSingleChar},
-		{"/", ErrorBadSpreadNotSingleChar},
-		{"a/", ErrorBadSpreadResultTooShort},
-		{"a/b/c/ab", ErrorBadSpreadResultTooShort},
-		{"a/b/x/abc", ErrorBadSpreadDidntMatch},
-		{"a/b/c/ABC", nil},
-	} {
-		actual, actualErr := unspreadName(test.in)
-		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
-	}
-}
-
-func TestEncryptName(t *testing.T) {
-	// First no flatten
-	c, _ := newCipher(0, "", "")
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptName("1"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptName("1/12"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptName("1/12/123"))
-	// Now with flatten
-	c, _ = newCipher(3, "", "")
-	assert.Equal(t, "k/g/t/kgtickdcigo7600huebjl3ubu4", c.EncryptName("1/12/123"))
-}
-
-func TestDecryptName(t *testing.T) {
-	for _, test := range []struct {
-		flatten     int
+		mode        NameEncryptionMode
 		in          string
 		expected    string
 		expectedErr error
 	}{
-		{0, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
-		{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
-		{0, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
-		{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
-		{3, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
-		{1, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
-		{1, "k/g/t/i/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil},
-		{1, "k/x/t/i/kgtickdcigo7600huebjl3ubu4", "", ErrorBadSpreadDidntMatch},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
+		{NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
+		{NameEncryptionOff, "1/12/123.bin", "1/12/123", nil},
+		{NameEncryptionOff, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
+		{NameEncryptionOff, ".bin", "", ErrorNotAnEncryptedFile},
 	} {
-		c, _ := newCipher(test.flatten, "", "")
-		actual, actualErr := c.DecryptName(test.in)
-		what := fmt.Sprintf("Testing %q (flatten=%d)", test.in, test.flatten)
+		c, _ := newCipher(test.mode, "", "")
+		actual, actualErr := c.DecryptFileName(test.in)
+		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+		assert.Equal(t, test.expected, actual, what)
+		assert.Equal(t, test.expectedErr, actualErr, what)
+	}
+}
+
+func TestEncryptDirName(t *testing.T) {
+	// First standard mode
+	c, _ := newCipher(NameEncryptionStandard, "", "")
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12"))
+	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123"))
+	// Now off mode
+	c, _ = newCipher(NameEncryptionOff, "", "")
+	assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+}
+
+func TestDecryptDirName(t *testing.T) {
+	for _, test := range []struct {
+		mode        NameEncryptionMode
+		in          string
+		expected    string
+		expectedErr error
+	}{
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
+		{NameEncryptionStandard, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
+		{NameEncryptionStandard, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
+		{NameEncryptionOff, "1/12/123.bin", "1/12/123.bin", nil},
+		{NameEncryptionOff, "1/12/123", "1/12/123", nil},
+		{NameEncryptionOff, ".bin", ".bin", nil},
+	} {
+		c, _ := newCipher(test.mode, "", "")
+		actual, actualErr := c.DecryptDirName(test.in)
+		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
 		assert.Equal(t, test.expected, actual, what)
 		assert.Equal(t, test.expectedErr, actualErr, what)
 	}
 }
 
 func TestEncryptedSize(t *testing.T) {
-	c, _ := newCipher(0, "", "")
+	c, _ := newCipher(NameEncryptionStandard, "", "")
 	for _, test := range []struct {
 		in       int64
 		expected int64
@@ -288,7 +305,7 @@ func TestEncryptedSize(t *testing.T) {
 
 func TestDecryptedSize(t *testing.T) {
 	// Test the errors since we tested the reverse above
-	c, _ := newCipher(0, "", "")
+	c, _ := newCipher(NameEncryptionStandard, "", "")
 	for _, test := range []struct {
 		in          int64
 		expectedErr error
@@ -521,7 +538,7 @@ func (z *zeroes) Read(p []byte) (n int, err error) {
 
 // Test encrypt decrypt with different buffer sizes
 func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 	c.cryptoRand = &zeroes{} // zero out the nonce
 	buf := make([]byte, bufSize)
@@ -591,7 +608,7 @@ func TestEncryptData(t *testing.T) {
 		{[]byte{1}, file1},
 		{[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16},
 	} {
-		c, err := newCipher(0, "", "")
+		c, err := newCipher(NameEncryptionStandard, "", "")
 		assert.NoError(t, err)
 		c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
@@ -614,7 +631,7 @@ func TestEncryptData(t *testing.T) {
 }
 
 func TestNewEncrypter(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
@@ -658,7 +675,7 @@ func (c *closeDetector) Close() error {
 }
 
 func TestNewDecrypter(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
 
@@ -700,7 +717,7 @@ func TestNewDecrypter(t *testing.T) {
 }
 
 func TestDecrypterRead(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 
 	// Test truncating the header
@@ -744,7 +761,7 @@ func TestDecrypterRead(t *testing.T) {
 }
 
 func TestDecrypterClose(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 
 	cd := newCloseDetector(bytes.NewBuffer(file16))
@@ -780,7 +797,7 @@ func TestDecrypterClose(t *testing.T) {
 }
 
 func TestPutGetBlock(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 
 	block := c.getBlock()
@@ -791,7 +808,7 @@ func TestPutGetBlock(t *testing.T) {
 }
 
 func TestKey(t *testing.T) {
-	c, err := newCipher(0, "", "")
+	c, err := newCipher(NameEncryptionStandard, "", "")
 	assert.NoError(t, err)
 
 	// Check zero keys OK

crypt/crypt.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"path"
-	"strings"
 	"sync"
 
 	"github.com/ncw/rclone/fs"
@@ -22,27 +21,15 @@ func init() {
 			Name: "remote",
 			Help: "Remote to encrypt/decrypt.",
 		}, {
-			Name: "flatten",
-			Help: "Flatten the directory structure - more secure, less useful - see docs for tradeoffs.",
+			Name: "filename_encryption",
+			Help: "How to encrypt the filenames.",
 			Examples: []fs.OptionExample{
 				{
-					Value: "0",
-					Help:  "Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.",
+					Value: "off",
+					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
 				}, {
-					Value: "1",
-					Help:  "Spread files over 1 directory good for <10,000 files.",
-				}, {
-					Value: "2",
-					Help:  "Spread files over 32 directories good for <320,000 files.",
-				}, {
-					Value: "3",
-					Help:  "Spread files over 1024 directories good for <10,000,000 files.",
-				}, {
-					Value: "4",
-					Help:  "Spread files over 32,768 directories good for <320,000,000 files.",
-				}, {
-					Value: "5",
-					Help:  "Spread files over 1,048,576 levels good for <10,000,000,000 files.",
+					Value: "standard",
+					Help:  "Encrypt the filenames see the docs for the details.",
 				},
 			},
 		}, {
@@ -60,12 +47,15 @@ func init() {
 
 // NewFs contstructs an Fs from the path, container:path
 func NewFs(name, rpath string) (fs.Fs, error) {
-	flatten := fs.ConfigFile.MustInt(name, "flatten", 0)
+	mode, err := NewNameEncryptionMode(fs.ConfigFile.MustValue(name, "filename_encryption", "standard"))
+	if err != nil {
+		return nil, err
+	}
 	password := fs.ConfigFile.MustValue(name, "password", "")
 	if password == "" {
 		return nil, errors.New("password not set in config file")
 	}
-	password, err := fs.Reveal(password)
+	password, err = fs.Reveal(password)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to decrypt password")
 	}
@@ -76,20 +66,26 @@ func NewFs(name, rpath string) (fs.Fs, error) {
 			return nil, errors.Wrap(err, "failed to decrypt password2")
 		}
 	}
-	cipher, err := newCipher(flatten, password, salt)
+	cipher, err := newCipher(mode, password, salt)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make cipher")
 	}
 	remote := fs.ConfigFile.MustValue(name, "remote")
-	remotePath := path.Join(remote, cipher.EncryptName(rpath))
+	// Look for a file first
+	remotePath := path.Join(remote, cipher.EncryptFileName(rpath))
 	wrappedFs, err := fs.NewFs(remotePath)
+	// if that didn't produce a file, look for a directory
+	if err != fs.ErrorIsFile {
+		remotePath = path.Join(remote, cipher.EncryptDirName(rpath))
+		wrappedFs, err = fs.NewFs(remotePath)
+	}
 	if err != fs.ErrorIsFile && err != nil {
 		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath)
 	}
 	f := &Fs{
-		Fs:      wrappedFs,
-		cipher:  cipher,
-		flatten: flatten,
+		Fs:     wrappedFs,
+		cipher: cipher,
+		mode:   mode,
 	}
 	return f, err
 }
@@ -97,8 +93,8 @@ func NewFs(name, rpath string) (fs.Fs, error) {
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
-	cipher  Cipher
-	flatten int
+	cipher Cipher
+	mode   NameEncryptionMode
 }
 
 // String returns a description of the FS
@@ -108,12 +104,12 @@ func (f *Fs) String() string {
 
 // List the Fs into a channel
 func (f *Fs) List(opts fs.ListOpts, dir string) {
-	f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptName(dir))
+	f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir))
 }
 
 // NewObject finds the Object at remote.
 func (f *Fs) NewObject(remote string) (fs.Object, error) {
-	o, err := f.Fs.NewObject(f.cipher.EncryptName(remote))
+	o, err := f.Fs.NewObject(f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
@@ -174,7 +170,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	if !ok {
 		return nil, fs.ErrorCantCopy
 	}
-	oResult, err := do.Copy(o.Object, f.cipher.EncryptName(remote))
+	oResult, err := do.Copy(o.Object, f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
@@ -199,7 +195,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	if !ok {
 		return nil, fs.ErrorCantCopy
 	}
-	oResult, err := do.Move(o.Object, f.cipher.EncryptName(remote))
+	oResult, err := do.Move(o.Object, f.cipher.EncryptFileName(remote))
 	if err != nil {
 		return nil, err
 	}
@@ -242,7 +238,7 @@ func (o *Object) String() string {
 // Remote returns the remote path
 func (o *Object) Remote() string {
 	remote := o.Object.Remote()
-	decryptedName, err := o.f.cipher.DecryptName(remote)
+	decryptedName, err := o.f.cipher.DecryptFileName(remote)
 	if err != nil {
 		fs.Debug(remote, "Undecryptable file name: %v", err)
 		return remote
@@ -287,7 +283,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
 func (f *Fs) newDir(dir *fs.Dir) *fs.Dir {
 	new := *dir
 	remote := dir.Name
-	decryptedRemote, err := f.cipher.DecryptName(remote)
+	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debug(remote, "Undecryptable dir name: %v", err)
 	} else {
@@ -318,7 +314,7 @@ func (o *ObjectInfo) Fs() fs.Info {
 
 // Remote returns the remote path
 func (o *ObjectInfo) Remote() string {
-	return o.f.cipher.EncryptName(o.ObjectInfo.Remote())
+	return o.f.cipher.EncryptFileName(o.ObjectInfo.Remote())
 }
 
 // Size returns the size of the file
@@ -358,55 +354,19 @@ func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts {
 //
 // Each returned item must have less than level `/`s in.
 func (lo *ListOpts) Level() int {
-	// If flattened recurse fully
-	if lo.f.flatten > 0 {
-		return fs.MaxLevel
-	}
 	return lo.ListOpts.Level()
 }
 
-// addSyntheticDirs makes up directory objects for the path passed in
-func (lo *ListOpts) addSyntheticDirs(path string) {
-	lo.mu.Lock()
-	defer lo.mu.Unlock()
-	for {
-		i := strings.LastIndexByte(path, '/')
-		if i < 0 {
-			break
-		}
-		path = path[:i]
-		if path == "" {
-			break
-		}
-		if _, found := lo.dirs[path]; found {
-			break
-		}
-		slashes := strings.Count(path, "/")
-		if slashes < lo.ListOpts.Level() {
-			lo.ListOpts.AddDir(&fs.Dir{Name: path})
-		}
-		lo.dirs[path] = struct{}{}
-	}
-}
-
 // Add an object to the output.
 // If the function returns true, the operation has been aborted.
 // Multiple goroutines can safely add objects concurrently.
 func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
 	remote := obj.Remote()
-	decryptedRemote, err := lo.f.cipher.DecryptName(remote)
+	_, err := lo.f.cipher.DecryptFileName(remote)
 	if err != nil {
 		fs.Debug(remote, "Skipping undecryptable file name: %v", err)
 		return lo.ListOpts.IsFinished()
 	}
-	// If flattened add synthetic directories
-	if lo.f.flatten > 0 {
-		lo.addSyntheticDirs(decryptedRemote)
-		slashes := strings.Count(decryptedRemote, "/")
-		if slashes >= lo.ListOpts.Level() {
-			return lo.ListOpts.IsFinished()
-		}
-	}
 	return lo.ListOpts.Add(lo.f.newObject(obj))
 }
 
@@ -414,12 +374,8 @@ func (lo *ListOpts) Add(obj fs.Object) (abort bool) {
 // If the function returns true, the operation has been aborted.
 // Multiple goroutines can safely add objects concurrently.
 func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
-	// If flattened we don't add any directories from the underlying remote
-	if lo.f.flatten > 0 {
-		return lo.ListOpts.IsFinished()
-	}
 	remote := dir.Name
-	_, err := lo.f.cipher.DecryptName(remote)
+	_, err := lo.f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debug(remote, "Skipping undecryptable dir name: %v", err)
 		return lo.ListOpts.IsFinished()
@@ -430,11 +386,7 @@ func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) {
 // IncludeDirectory returns whether this directory should be
 // included in the listing (and recursed into or not).
 func (lo *ListOpts) IncludeDirectory(remote string) bool {
-	// If flattened we look in all directories
-	if lo.f.flatten > 0 {
-		return true
-	}
-	decryptedRemote, err := lo.f.cipher.DecryptName(remote)
+	decryptedRemote, err := lo.f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debug(remote, "Not including undecryptable directory name: %v", err)
 		return false

docs/content/crypt.md

@@ -19,8 +19,8 @@ First check your chosen remote is working - we'll call it
 will be encrypted and anything outside won't.  This means that if you
 are using a bucket based remote (eg S3, B2, swift) then you should
 probably put the bucket in the remote `s3:bucket`. If you just use
-`s3:` then rclone will make encrypted bucket names too which may or
-may not be what you want.
+`s3:` then rclone will make encrypted bucket names too (if using file
+name encryption) which may or may not be what you want.
 
 Now configure `crypt` using `rclone config`. We will call this one
 `secret` to differentiate it from the `remote`.
@@ -30,7 +30,7 @@ No remotes found - make a new one
 n) New remote
 s) Set configuration password
 q) Quit config
-n/s/q> n
+n/s/q> n   
 name> secret
 Type of storage to configure.
 Choose a number from below, or type in your own value
@@ -61,32 +61,44 @@ Choose a number from below, or type in your own value
 Storage> 5
 Remote to encrypt/decrypt.
 remote> remote:path
-Flatten the directory structure - more secure, less useful - see docs for tradeoffs.
+How to encrypt the filenames.
 Choose a number from below, or type in your own value
- 1 / Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.
-   \ "0"
- 2 / Spread files over 1 directory good for <10,000 files.
-   \ "1"
- 3 / Spread files over 32 directories good for <320,000 files.
-   \ "2"
- 4 / Spread files over 1024 directories good for <10,000,000 files.
-   \ "3"
- 5 / Spread files over 32,768 directories good for <320,000,000 files.
-   \ "4"
- 6 / Spread files over 1,048,576 levels good for <10,000,000,000 files.
-   \ "5"
-flatten> 1
+ 1 / Don't encrypt the file names.  Adds a ".bin" extension only.
+   \ "off"
+ 2 / Encrypt the filenames see the docs for the details.
+   \ "standard"
+filename_encryption> 2
 Password or pass phrase for encryption.
+y) Yes type in my own password
+g) Generate random password
+y/g> y
 Enter the password:
 password:
 Confirm the password:
 password:
+Password or pass phrase for salt. Optional but recommended.
+Should be different to the previous password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> g
+Password strength in bits.
+64 is just about memorable
+128 is secure
+1024 is the maximum
+Bits> 128
+Your password is: JAsJvRcgR-_veXNfy_sGmQ
+Use this password?
+y) Yes
+n) No
+y/n> y
 Remote config
 --------------------
 [secret]
 remote = remote:path
-flatten = 0
-password = 0_gtCJ422bzwAWP0UN2lggrjhA-sSg
+filename_encryption = standard
+password = CfDxopZIXFG0Oo-ac7dPLWWOHkNJbw
+password2 = HYUpfuzHJL8qnX9fOaIYijq0xnVLwyVzp3y4SF3TwYqAU6HLysk
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -99,9 +111,9 @@ obscured so it isn't immediately obvious what it is.  It is in no way
 secure unless you use config file encryption.
 
 A long passphrase is recommended, or you can use a random one.  Note
-that if you reconfigure rclone with the same password/passphrase
+that if you reconfigure rclone with the same passwords/passphrases
 elsewhere it will be compatible - all the secrets used are derived
-from that one password/passphrase.
+from those two passwords/passphrases.
 
 Note that rclone does not encrypt
   * file length - this can be calcuated within 16 bytes
@@ -109,7 +121,8 @@ Note that rclone does not encrypt
 
 ## Example ##
 
-To test I made a little directory of files
+To test I made a little directory of files using "standard" file name
+encryption.
 
 ```
 plaintext/
@@ -154,39 +167,43 @@ $ rclone -q ls secret:subdir
        10 subsubdir/file4.txt
 ```
 
-If you use the flattened flag then the listing will look and that last command will not work.
+If don't use file name encryption then the remote will look like this
+- note the `.bin` extensions added to prevent the cloud provider
+attempting to interpret the data.
 
 ```
 $ rclone -q ls remote:path
-       56 t/tsdtcpdu6g9dpamn6poqc248tll9dj5ok78a363etmq8ushr821g
-       57 g/gsrp2g0u85pgsi6kso74bjsrsafe11odpfln8qqpj6n9p20of0a0
-       55 h/hagjclgavj2mbiqm6u6cnjjqcg
-       58 4/4jsbao3dhi0jfoubt2oo493pbqmsshn92q01ddu7dg6428rlluhg
-       54 v/v05749mltvv1tf4onltun46gls
+       54 file0.txt.bin
+       57 subdir/file3.txt.bin
+       56 subdir/file2.txt.bin
+       58 subdir/subsubdir/file4.txt.bin
+       55 file1.txt.bin
 ```
 
-### Flattened vs non-Flattened ###
+### File name encryption modes ###
 
-Pros and cons of each
+Here are some of the features of the file name encryption modes
 
-Flattened
-  * hides directory structures
-  * identical file names won't have identical encrypted names
-  * can't use a sub path
-    * doesn't work: `rclone copy crypt:sub/dir /tmp/recovered`
-    * use: `rclone copy --include "/sub/dir/**" crypt: /tmp/recovered`
-  * will always have to recurse through the entire directory structure
-  * can't copy a single file directly
-    * doesn't work: `rclone copy crypt:path/to/file /tmp/recovered`
-    * use: `rclone copy --include "/path/to/file" crypt: /tmp/recovered`
+Off
+  * doesn't hide file names or directory structure
+  * allows for longer file names (~246 characters)
+  * can use sub paths and copy single files
 
-Normal
+Standard
+  * file names encrypted
+  * file names can't be as long (~156 characters)
   * can use sub paths and copy single files
   * directory structure visibile
   * identical files names will have identical uploaded names
   * can use shortcuts to shorten the directory recursion
 
-You can swap between flattened levels without re-uploading your files.
+Cloud storage systems have various limits on file name length and
+total path length which you are more likely to hit using "Standard"
+file name encryption.  If you keep your file names to below 156
+characters in length then you should be OK on all providers.
+
+There may be an even more secure file name encryption mode in the
+future which will address the long file name problem.
 
 ## File formats ##
 
@@ -245,25 +262,23 @@ files.
 
 ### Name encryption ###
 
-File names are encrypted by crypt.  These are either encrypted segment
-by segment - the path is broken up into `/` separated strings and
-these are encrypted individually, or if working in flattened mode the
-whole path is encrypted `/`s and all.
+File names are encrypted segment by segment - the path is broken up
+into `/` separated strings and these are encrypted individually.
 
-First file names are padded using using PKCS#7 to a multiple of 16
-bytes before encryption.
+File segments are padded using using PKCS#7 to a multiple of 16 bytes
+before encryption.
 
 They are then encrypted with EME using AES with 256 bit key. EME
 (ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003
 paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway.
 
 This makes for determinstic encryption which is what we want - the
-same filename must encrypt to the same thing.
+same filename must encrypt to the same thing otherwise we can't find
+it on the cloud storage system.
 
 This means that
 
   * filenames with the same name will encrypt the same
-    * (though we can use directory flattening to avoid this if required)
   * filenames which start the same won't have a common prefix
 
 This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of
@@ -281,8 +296,11 @@ used on case insensitive remotes (eg Windows, Amazon Drive).
 
 ### Key derivation ###
 
-Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a fixed
-salt to derive the 32+32+16 = 80 bytes of key material required.
+Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a an
+optional user supplied salt (password2) to derive the 32+32+16 = 80
+bytes of key material required.  If the user doesn't supply a salt
+then rclone uses an internal one.
 
 `scrypt` makes it impractical to mount a dictionary attack on rclone
-encrypted data.
+encrypted data.  For full protection agains this you should always use
+a salt.