From 226c2a0d83d20895df3d2314d38115c3da6e2e9b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 25 Jul 2016 19:18:56 +0100 Subject: [PATCH] Implement crypt for encrypted remotes - #219 --- crypt/cipher.go | 574 +++++++++++++++++++++++++ crypt/cipher_test.go | 816 ++++++++++++++++++++++++++++++++++++ crypt/crypt.go | 445 ++++++++++++++++++++ crypt/crypt_test.go | 59 +++ crypt/pkcs7/pkcs7.go | 63 +++ crypt/pkcs7/pkcs7_test.go | 73 ++++ docs/content/crypt.md | 288 +++++++++++++ fs/all/all.go | 1 + fstest/fstests/fstests.go | 15 +- fstest/fstests/gen_tests.go | 4 +- 10 files changed, 2333 insertions(+), 5 deletions(-) create mode 100644 crypt/cipher.go create mode 100644 crypt/cipher_test.go create mode 100644 crypt/crypt.go create mode 100644 crypt/crypt_test.go create mode 100644 crypt/pkcs7/pkcs7.go create mode 100644 crypt/pkcs7/pkcs7_test.go create mode 100644 docs/content/crypt.md diff --git a/crypt/cipher.go b/crypt/cipher.go new file mode 100644 index 000000000..7e358902c --- /dev/null +++ b/crypt/cipher.go @@ -0,0 +1,574 @@ +package crypt + +import ( + "bytes" + "crypto/aes" + gocipher "crypto/cipher" + "crypto/rand" + "encoding/base32" + "io" + "strings" + "sync" + "unicode/utf8" + + "github.com/ncw/rclone/crypt/pkcs7" + "github.com/pkg/errors" + + "golang.org/x/crypto/nacl/secretbox" + "golang.org/x/crypto/scrypt" + + "github.com/rfjakob/eme" +) + +// Constancs +const ( + nameCipherBlockSize = aes.BlockSize + fileMagic = "RCLONE\x00\x00" + fileMagicSize = len(fileMagic) + fileNonceSize = 24 + fileHeaderSize = fileMagicSize + fileNonceSize + blockHeaderSize = secretbox.Overhead + blockDataSize = 64 * 1024 + blockSize = blockHeaderSize + blockDataSize +) + +// Errors returned by cipher +var ( + ErrorBadDecryptUTF8 = errors.New("bad decryption - utf-8 invalid") + ErrorBadDecryptControlChar = errors.New("bad decryption - contains control chars") + ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize") + ErrorTooShortAfterDecode = errors.New("too short after base32 decode") + ErrorEncryptedFileTooShort = errors.New("file is too short to be encrypted") + ErrorEncryptedFileBadHeader = errors.New("file has truncated block header") + ErrorEncryptedBadMagic = errors.New("not an encrypted file - bad magic string") + ErrorEncryptedBadBlock = errors.New("failed to authenticate decrypted block - bad password?") + ErrorBadBase32Encoding = errors.New("bad base32 filename encoding") + ErrorBadSpreadNotSingleChar = errors.New("bad unspread - not single character") + ErrorBadSpreadResultTooShort = errors.New("bad unspread - result too short") + ErrorBadSpreadDidntMatch = errors.New("bad unspread - directory prefix didn't match") + ErrorFileClosed = errors.New("file already closed") + scryptSalt = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1} +) + +// Global variables +var ( + fileMagicBytes = []byte(fileMagic) +) + +// Cipher is used to swap out the encryption implementations +type Cipher interface { + // EncryptName encrypts a file path + EncryptName(string) string + // DecryptName decrypts a file path, returns error if decrypt was invalid + DecryptName(string) (string, error) + // EncryptData + EncryptData(io.Reader) (io.Reader, error) + // DecryptData + DecryptData(io.ReadCloser) (io.ReadCloser, error) + // EncryptedSize calculates the size of the data when encrypted + EncryptedSize(int64) int64 + // DecryptedSize calculates the size of the data when decrypted + DecryptedSize(int64) (int64, error) +} + +type cipher struct { + dataKey [32]byte // Key for secretbox + nameKey [32]byte // 16,24 or 32 bytes + nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto + block gocipher.Block + flatten int // set flattening level - 0 is off + buffers sync.Pool // encrypt/decrypt buffers + cryptoRand io.Reader // read crypto random numbers from here +} + +func newCipher(flatten int, password string) (*cipher, error) { + c := &cipher{ + flatten: flatten, + cryptoRand: rand.Reader, + } + c.buffers.New = func() interface{} { + return make([]byte, blockSize) + } + err := c.Key(password) + if err != nil { + return nil, err + } + return c, nil +} + +// Key creates all the internal keys from the password passed in using +// scrypt. We use a fixed salt just to make attackers lives slighty +// harder than using no salt. +// +// Note that empty passsword makes all 0x00 keys which is used in the +// tests. +func (c *cipher) Key(password string) (err error) { + const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak) + var key []byte + if password == "" { + key = make([]byte, keySize) + } else { + key, err = scrypt.Key([]byte(password), scryptSalt, 16384, 8, 1, keySize) + if err != nil { + return err + } + } + copy(c.dataKey[:], key) + copy(c.nameKey[:], key[len(c.dataKey):]) + copy(c.nameTweak[:], key[len(c.dataKey)+len(c.nameKey):]) + // Key the name cipher + c.block, err = aes.NewCipher(c.nameKey[:]) + return err +} + +// getBlock gets a block from the pool of size blockSize +func (c *cipher) getBlock() []byte { + return c.buffers.Get().([]byte) +} + +// putBlock returns a block to the pool of size blockSize +func (c *cipher) putBlock(buf []byte) { + if len(buf) != blockSize { + panic("bad blocksize returned to pool") + } + c.buffers.Put(buf) +} + +// check to see if the byte string is valid with no control characters +// from 0x00 to 0x1F and is a valid UTF-8 string +func checkValidString(buf []byte) error { + for i := range buf { + c := buf[i] + if c >= 0x00 && c < 0x20 || c == 0x7F { + return ErrorBadDecryptControlChar + } + } + if !utf8.Valid(buf) { + return ErrorBadDecryptUTF8 + } + return nil +} + +// encodeFileName encodes a filename using a modified version of +// standard base32 as described in RFC4648 +// +// The standard encoding is modified in two ways +// * it becomes lower case (no-one likes upper case filenames!) +// * we strip the padding character `=` +func encodeFileName(in []byte) string { + encoded := base32.HexEncoding.EncodeToString(in) + encoded = strings.TrimRight(encoded, "=") + return strings.ToLower(encoded) +} + +// decodeFileName decodes a filename as encoded by encodeFileName +func decodeFileName(in string) ([]byte, error) { + if strings.HasSuffix(in, "=") { + return nil, ErrorBadBase32Encoding + } + // First figure out how many padding characters to add + roundUpToMultipleOf8 := (len(in) + 7) &^ 7 + equals := roundUpToMultipleOf8 - len(in) + in = strings.ToUpper(in) + "========"[:equals] + return base32.HexEncoding.DecodeString(in) +} + +// encryptSegment encrypts a path segment +// +// This uses EME with AES +// +// EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the +// 2003 paper "A Parallelizable Enciphering Mode" by Halevi and +// Rogaway. +// +// This makes for determinstic encryption which is what we want - the +// same filename must encrypt to the same thing. +// +// This means that +// * filenames with the same name will encrypt the same +// * filenames which start the same won't have a common prefix +func (c *cipher) encryptSegment(plaintext string) string { + if plaintext == "" { + return "" + } + paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext)) + ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt) + return encodeFileName(ciphertext) +} + +// decryptSegment decrypts a path segment +func (c *cipher) decryptSegment(ciphertext string) (string, error) { + if ciphertext == "" { + return "", nil + } + rawCiphertext, err := decodeFileName(ciphertext) + if err != nil { + return "", err + } + if len(rawCiphertext)%nameCipherBlockSize != 0 { + return "", ErrorNotAMultipleOfBlocksize + } + if len(rawCiphertext) == 0 { + // not possible if decodeFilename() working correctly + return "", ErrorTooShortAfterDecode + } + paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt) + plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext) + if err != nil { + return "", err + } + err = checkValidString(plaintext) + if err != nil { + return "", err + } + return string(plaintext), err +} + +// spread a name over the given number of directory levels +// +// if in isn't long enough dirs will be reduces +func spreadName(dirs int, in string) string { + if dirs > len(in) { + dirs = len(in) + } + prefix := "" + for i := 0; i < dirs; i++ { + prefix += string(in[i]) + "/" + } + return prefix + in +} + +// reverse spreadName, returning an error if not in spread format +// +// This decodes any level of spreading +func unspreadName(in string) (string, error) { + in = strings.ToLower(in) + segments := strings.Split(in, "/") + if len(segments) == 0 { + return in, nil + } + out := segments[len(segments)-1] + segments = segments[:len(segments)-1] + for i, s := range segments { + if len(s) != 1 { + return "", ErrorBadSpreadNotSingleChar + } + if i >= len(out) { + return "", ErrorBadSpreadResultTooShort + } + if s[0] != out[i] { + return "", ErrorBadSpreadDidntMatch + } + } + return out, nil +} + +// EncryptName encrypts a file path +func (c *cipher) EncryptName(in string) string { + if c.flatten > 0 { + return spreadName(c.flatten, c.encryptSegment(in)) + } + segments := strings.Split(in, "/") + for i := range segments { + segments[i] = c.encryptSegment(segments[i]) + } + return strings.Join(segments, "/") +} + +// DecryptName decrypts a file path +func (c *cipher) DecryptName(in string) (string, error) { + if c.flatten > 0 { + unspread, err := unspreadName(in) + if err != nil { + return "", err + } + return c.decryptSegment(unspread) + } + segments := strings.Split(in, "/") + for i := range segments { + var err error + segments[i], err = c.decryptSegment(segments[i]) + if err != nil { + return "", err + } + } + return strings.Join(segments, "/"), nil +} + +// nonce is an NACL secretbox nonce +type nonce [fileNonceSize]byte + +// pointer returns the nonce as a *[24]byte for secretbox +func (n *nonce) pointer() *[fileNonceSize]byte { + return (*[fileNonceSize]byte)(n) +} + +// fromReader fills the nonce from an io.Reader - normally the OSes +// crypto random number generator +func (n *nonce) fromReader(in io.Reader) error { + read, err := io.ReadFull(in, (*n)[:]) + if read != fileNonceSize { + return errors.Wrap(err, "short read of nonce") + } + return nil +} + +// fromBuf fills the nonce from the buffer passed in +func (n *nonce) fromBuf(buf []byte) { + read := copy((*n)[:], buf) + if read != fileNonceSize { + panic("buffer to short to read nonce") + } +} + +// increment to add 1 to the nonce +func (n *nonce) increment() { + for i := 0; i < len(*n); i++ { + digit := (*n)[i] + newDigit := digit + 1 + (*n)[i] = newDigit + if newDigit >= digit { + // exit if no carry + break + } + } +} + +// encrypter encrypts an io.Reader on the fly +type encrypter struct { + in io.Reader + c *cipher + nonce nonce + buf []byte + readBuf []byte + bufIndex int + bufSize int + err error +} + +// newEncrypter creates a new file handle encrypting on the fly +func (c *cipher) newEncrypter(in io.Reader) (*encrypter, error) { + fh := &encrypter{ + in: in, + c: c, + buf: c.getBlock(), + readBuf: c.getBlock(), + bufSize: fileHeaderSize, + } + // Initialise nonce + err := fh.nonce.fromReader(c.cryptoRand) + if err != nil { + return nil, err + } + // Copy magic into buffer + copy(fh.buf, fileMagicBytes) + // Copy nonce into buffer + copy(fh.buf[fileMagicSize:], fh.nonce[:]) + return fh, nil +} + +// Read as per io.Reader +func (fh *encrypter) Read(p []byte) (n int, err error) { + if fh.err != nil { + return 0, fh.err + } + if fh.bufIndex >= fh.bufSize { + // Read data + // FIXME should overlap the reads with a go-routine and 2 buffers? + readBuf := fh.readBuf[:blockDataSize] + n, err = io.ReadFull(fh.in, readBuf) + if err == io.EOF { + // ReadFull only returns n=0 and EOF + return fh.finish(io.EOF) + } else if err == io.ErrUnexpectedEOF { + // Next read will return EOF + } else if err != nil { + return fh.finish(err) + } + // Write nonce to start of block + copy(fh.buf, fh.nonce[:]) + // Encrypt the block using the nonce + block := fh.buf + secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey) + fh.bufIndex = 0 + fh.bufSize = blockHeaderSize + n + fh.nonce.increment() + } + n = copy(p, fh.buf[fh.bufIndex:fh.bufSize]) + fh.bufIndex += n + return n, nil +} + +// finish sets the final error and tidies up +func (fh *encrypter) finish(err error) (int, error) { + if fh.err != nil { + return 0, fh.err + } + fh.err = err + fh.c.putBlock(fh.buf) + fh.c.putBlock(fh.readBuf) + return 0, err +} + +// Encrypt data encrypts the data stream +func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) { + out, err := c.newEncrypter(in) + if err != nil { + return nil, err + } + return out, nil +} + +// decrypter decrypts an io.ReaderCloser on the fly +type decrypter struct { + rc io.ReadCloser + nonce nonce + c *cipher + buf []byte + readBuf []byte + bufIndex int + bufSize int + err error +} + +// newDecrypter creates a new file handle decrypting on the fly +func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) { + fh := &decrypter{ + rc: rc, + c: c, + buf: c.getBlock(), + readBuf: c.getBlock(), + } + // Read file header (magic + nonce) + readBuf := fh.readBuf[:fileHeaderSize] + _, err := io.ReadFull(fh.rc, readBuf) + if err == io.EOF || err == io.ErrUnexpectedEOF { + // This read from 0..fileHeaderSize-1 bytes + return nil, fh.finishAndClose(ErrorEncryptedFileTooShort) + } else if err != nil { + return nil, fh.finishAndClose(err) + } + // check the magic + if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) { + return nil, fh.finishAndClose(ErrorEncryptedBadMagic) + } + // retreive the nonce + fh.nonce.fromBuf(readBuf[fileMagicSize:]) + return fh, nil +} + +// Read as per io.Reader +func (fh *decrypter) Read(p []byte) (n int, err error) { + if fh.err != nil { + return 0, fh.err + } + if fh.bufIndex >= fh.bufSize { + // Read data + // FIXME should overlap the reads with a go-routine and 2 buffers? + readBuf := fh.readBuf + n, err = io.ReadFull(fh.rc, readBuf) + if err == io.EOF { + // ReadFull only returns n=0 and EOF + return 0, fh.finish(io.EOF) + } else if err == io.ErrUnexpectedEOF { + // Next read will return EOF + } else if err != nil { + return 0, fh.finish(err) + } + // Check header + 1 byte exists + if n <= blockHeaderSize { + return 0, fh.finish(ErrorEncryptedFileBadHeader) + } + // Decrypt the block using the nonce + block := fh.buf + _, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey) + if !ok { + return 0, fh.finish(ErrorEncryptedBadBlock) + } + fh.bufIndex = 0 + fh.bufSize = n - blockHeaderSize + fh.nonce.increment() + } + n = copy(p, fh.buf[fh.bufIndex:fh.bufSize]) + fh.bufIndex += n + return n, nil +} + +// finish sets the final error and tidies up +func (fh *decrypter) finish(err error) error { + if fh.err != nil { + return fh.err + } + fh.err = err + fh.c.putBlock(fh.buf) + fh.c.putBlock(fh.readBuf) + return err +} + +// Close +func (fh *decrypter) Close() error { + // Check already closed + if fh.err == ErrorFileClosed { + return fh.err + } + // Closed before reading EOF so not finish()ed yet + if fh.err == nil { + _ = fh.finish(io.EOF) + } + // Show file now closed + fh.err = ErrorFileClosed + return fh.rc.Close() +} + +// finishAndClose does finish then Close() +// +// Used when we are returning a nil fh from new +func (fh *decrypter) finishAndClose(err error) error { + _ = fh.finish(err) + _ = fh.Close() + return err +} + +// DecryptData decrypts the data stream +func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) { + out, err := c.newDecrypter(rc) + if err != nil { + return nil, err + } + return out, nil +} + +// EncryptedSize calculates the size of the data when encrypted +func (c *cipher) EncryptedSize(size int64) int64 { + blocks, residue := size/blockDataSize, size%blockDataSize + encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) + if residue != 0 { + encryptedSize += blockHeaderSize + residue + } + return encryptedSize +} + +// DecryptedSize calculates the size of the data when decrypted +func (c *cipher) DecryptedSize(size int64) (int64, error) { + size -= int64(fileHeaderSize) + if size < 0 { + return 0, ErrorEncryptedFileTooShort + } + blocks, residue := size/blockSize, size%blockSize + decryptedSize := blocks * blockDataSize + if residue != 0 { + residue -= blockHeaderSize + if residue <= 0 { + return 0, ErrorEncryptedFileBadHeader + } + } + decryptedSize += residue + return decryptedSize, nil +} + +// check interfaces +var ( + _ Cipher = (*cipher)(nil) + _ io.ReadCloser = (*decrypter)(nil) + _ io.Reader = (*encrypter)(nil) +) diff --git a/crypt/cipher_test.go b/crypt/cipher_test.go new file mode 100644 index 000000000..62140edf0 --- /dev/null +++ b/crypt/cipher_test.go @@ -0,0 +1,816 @@ +package crypt + +import ( + "bytes" + "encoding/base32" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" + + "github.com/ncw/rclone/crypt/pkcs7" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidString(t *testing.T) { + for _, test := range []struct { + in string + expected error + }{ + {"", nil}, + {"\x01", ErrorBadDecryptControlChar}, + {"a\x02", ErrorBadDecryptControlChar}, + {"abc\x03", ErrorBadDecryptControlChar}, + {"abc\x04def", ErrorBadDecryptControlChar}, + {"\x05d", ErrorBadDecryptControlChar}, + {"\x06def", ErrorBadDecryptControlChar}, + {"\x07", ErrorBadDecryptControlChar}, + {"\x08", ErrorBadDecryptControlChar}, + {"\x09", ErrorBadDecryptControlChar}, + {"\x0A", ErrorBadDecryptControlChar}, + {"\x0B", ErrorBadDecryptControlChar}, + {"\x0C", ErrorBadDecryptControlChar}, + {"\x0D", ErrorBadDecryptControlChar}, + {"\x0E", ErrorBadDecryptControlChar}, + {"\x0F", ErrorBadDecryptControlChar}, + {"\x10", ErrorBadDecryptControlChar}, + {"\x11", ErrorBadDecryptControlChar}, + {"\x12", ErrorBadDecryptControlChar}, + {"\x13", ErrorBadDecryptControlChar}, + {"\x14", ErrorBadDecryptControlChar}, + {"\x15", ErrorBadDecryptControlChar}, + {"\x16", ErrorBadDecryptControlChar}, + {"\x17", ErrorBadDecryptControlChar}, + {"\x18", ErrorBadDecryptControlChar}, + {"\x19", ErrorBadDecryptControlChar}, + {"\x1A", ErrorBadDecryptControlChar}, + {"\x1B", ErrorBadDecryptControlChar}, + {"\x1C", ErrorBadDecryptControlChar}, + {"\x1D", ErrorBadDecryptControlChar}, + {"\x1E", ErrorBadDecryptControlChar}, + {"\x1F", ErrorBadDecryptControlChar}, + {"\x20", nil}, + {"\x7E", nil}, + {"\x7F", ErrorBadDecryptControlChar}, + {"£100", nil}, + {`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil}, + {"£100", nil}, + // Following tests from http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805 + {"a", nil}, // Valid ASCII + {"\xc3\xb1", nil}, // Valid 2 Octet Sequence + {"\xc3\x28", ErrorBadDecryptUTF8}, // Invalid 2 Octet Sequence + {"\xa0\xa1", ErrorBadDecryptUTF8}, // Invalid Sequence Identifier + {"\xe2\x82\xa1", nil}, // Valid 3 Octet Sequence + {"\xe2\x28\xa1", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 2nd Octet) + {"\xe2\x82\x28", ErrorBadDecryptUTF8}, // Invalid 3 Octet Sequence (in 3rd Octet) + {"\xf0\x90\x8c\xbc", nil}, // Valid 4 Octet Sequence + {"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 2nd Octet) + {"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 3rd Octet) + {"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8}, // Invalid 4 Octet Sequence (in 4th Octet) + {"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 5 Octet Sequence (but not Unicode!) + {"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!) + } { + actual := checkValidString([]byte(test.in)) + assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in)) + } +} + +func TestEncodeFileName(t *testing.T) { + for _, test := range []struct { + in string + expected string + }{ + {"", ""}, + {"1", "64"}, + {"12", "64p0"}, + {"123", "64p36"}, + {"1234", "64p36d0"}, + {"12345", "64p36d1l"}, + {"123456", "64p36d1l6o"}, + {"1234567", "64p36d1l6org"}, + {"12345678", "64p36d1l6orjg"}, + {"123456789", "64p36d1l6orjge8"}, + {"1234567890", "64p36d1l6orjge9g"}, + {"12345678901", "64p36d1l6orjge9g64"}, + {"123456789012", "64p36d1l6orjge9g64p0"}, + {"1234567890123", "64p36d1l6orjge9g64p36"}, + {"12345678901234", "64p36d1l6orjge9g64p36d0"}, + {"123456789012345", "64p36d1l6orjge9g64p36d1l"}, + {"1234567890123456", "64p36d1l6orjge9g64p36d1l6o"}, + } { + actual := encodeFileName([]byte(test.in)) + assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in)) + recovered, err := decodeFileName(test.expected) + assert.NoError(t, err) + assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected)) + in := strings.ToUpper(test.expected) + recovered, err = decodeFileName(in) + assert.NoError(t, err) + assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in)) + } +} + +func TestDecodeFileName(t *testing.T) { + // We've tested decoding the valid ones above, now concentrate on the invalid ones + for _, test := range []struct { + in string + expectedErr error + }{ + {"64=", ErrorBadBase32Encoding}, + {"!", base32.CorruptInputError(0)}, + {"hello=hello", base32.CorruptInputError(5)}, + } { + actual, actualErr := decodeFileName(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) + } +} + +func TestEncryptSegment(t *testing.T) { + c, _ := newCipher(0, "") + for _, test := range []struct { + in string + expected string + }{ + {"", ""}, + {"1", "p0e52nreeaj0a5ea7s64m4j72s"}, + {"12", "l42g6771hnv3an9cgc8cr2n1ng"}, + {"123", "qgm4avr35m5loi1th53ato71v0"}, + {"1234", "8ivr2e9plj3c3esisjpdisikos"}, + {"12345", "rh9vu63q3o29eqmj4bg6gg7s44"}, + {"123456", "bn717l3alepn75b2fb2ejmi4b4"}, + {"1234567", "n6bo9jmb1qe3b1ogtj5qkf19k8"}, + {"12345678", "u9t24j7uaq94dh5q53m3s4t9ok"}, + {"123456789", "37hn305g6j12d1g0kkrl7ekbs4"}, + {"1234567890", "ot8d91eplaglb62k2b1trm2qv0"}, + {"12345678901", "h168vvrgb53qnrtvvmb378qrcs"}, + {"123456789012", "s3hsdf9e29ithrqbjqu01t8q2s"}, + {"1234567890123", "cf3jimlv1q2oc553mv7s3mh3eo"}, + {"12345678901234", "moq0uqdlqrblrc5pa5u5c7hq9g"}, + {"123456789012345", "eeam3li4rnommi3a762h5n7meg"}, + {"1234567890123456", "mijbj0frqf6ms7frcr6bd9h0env53jv96pjaaoirk7forcgpt70g"}, + } { + actual := c.encryptSegment(test.in) + assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in)) + recovered, err := c.decryptSegment(test.expected) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected)) + in := strings.ToUpper(test.expected) + recovered, err = c.decryptSegment(in) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in)) + } +} + +func TestDecryptSegment(t *testing.T) { + // We've tested the forwards above, now concentrate on the errors + c, _ := newCipher(0, "") + for _, test := range []struct { + in string + expectedErr error + }{ + {"64=", ErrorBadBase32Encoding}, + {"!", base32.CorruptInputError(0)}, + {encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize}, + {encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize}, + {encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong}, + {c.encryptSegment("\x01"), ErrorBadDecryptControlChar}, + {c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8}, + } { + actual, actualErr := c.decryptSegment(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) + } +} + +func TestSpreadName(t *testing.T) { + for _, test := range []struct { + n int + in string + expected string + }{ + {3, "", ""}, + {0, "abcdefg", "abcdefg"}, + {1, "abcdefg", "a/abcdefg"}, + {2, "abcdefg", "a/b/abcdefg"}, + {3, "abcdefg", "a/b/c/abcdefg"}, + {4, "abcdefg", "a/b/c/d/abcdefg"}, + {4, "abcd", "a/b/c/d/abcd"}, + {4, "abc", "a/b/c/abc"}, + {4, "ab", "a/b/ab"}, + {4, "a", "a/a"}, + } { + actual := spreadName(test.n, test.in) + assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d,%q", test.n, test.in)) + recovered, err := unspreadName(test.expected) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected)) + } +} + +func TestUnspreadName(t *testing.T) { + // We've tested the forwards above, now concentrate on the errors + for _, test := range []struct { + in string + expectedErr error + }{ + {"aa/bc", ErrorBadSpreadNotSingleChar}, + {"/", ErrorBadSpreadNotSingleChar}, + {"a/", ErrorBadSpreadResultTooShort}, + {"a/b/c/ab", ErrorBadSpreadResultTooShort}, + {"a/b/x/abc", ErrorBadSpreadDidntMatch}, + {"a/b/c/ABC", nil}, + } { + actual, actualErr := unspreadName(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr)) + } +} + +func TestEncryptName(t *testing.T) { + // First no flatten + c, _ := newCipher(0, "") + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptName("1")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptName("1/12")) + assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptName("1/12/123")) + // Now with flatten + c, _ = newCipher(3, "") + assert.Equal(t, "k/g/t/kgtickdcigo7600huebjl3ubu4", c.EncryptName("1/12/123")) +} + +func TestDecryptName(t *testing.T) { + for _, test := range []struct { + flatten int + in string + expected string + expectedErr error + }{ + {0, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil}, + {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil}, + {0, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil}, + {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil}, + {0, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize}, + {3, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, + {1, "k/g/t/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, + {1, "k/g/t/i/kgtickdcigo7600huebjl3ubu4", "1/12/123", nil}, + {1, "k/x/t/i/kgtickdcigo7600huebjl3ubu4", "", ErrorBadSpreadDidntMatch}, + } { + c, _ := newCipher(test.flatten, "") + actual, actualErr := c.DecryptName(test.in) + what := fmt.Sprintf("Testing %q (flatten=%d)", test.in, test.flatten) + assert.Equal(t, test.expected, actual, what) + assert.Equal(t, test.expectedErr, actualErr, what) + } +} + +func TestEncryptedSize(t *testing.T) { + c, _ := newCipher(0, "") + for _, test := range []struct { + in int64 + expected int64 + }{ + {0, 32}, + {1, 32 + 16 + 1}, + {65536, 32 + 16 + 65536}, + {65537, 32 + 16 + 65536 + 16 + 1}, + {1 << 20, 32 + 16*(16+65536)}, + {(1 << 20) + 65535, 32 + 16*(16+65536) + 16 + 65535}, + {1 << 30, 32 + 16384*(16+65536)}, + {(1 << 40) + 1, 32 + 16777216*(16+65536) + 16 + 1}, + } { + actual := c.EncryptedSize(test.in) + assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %d", test.in)) + recovered, err := c.DecryptedSize(test.expected) + assert.NoError(t, err, fmt.Sprintf("Testing reverse %d", test.expected)) + assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %d", test.expected)) + } +} + +func TestDecryptedSize(t *testing.T) { + // Test the errors since we tested the reverse above + c, _ := newCipher(0, "") + for _, test := range []struct { + in int64 + expectedErr error + }{ + {0, ErrorEncryptedFileTooShort}, + {0, ErrorEncryptedFileTooShort}, + {1, ErrorEncryptedFileTooShort}, + {7, ErrorEncryptedFileTooShort}, + {32 + 1, ErrorEncryptedFileBadHeader}, + {32 + 16, ErrorEncryptedFileBadHeader}, + {32 + 16 + 65536 + 1, ErrorEncryptedFileBadHeader}, + {32 + 16 + 65536 + 16, ErrorEncryptedFileBadHeader}, + } { + _, actualErr := c.DecryptedSize(test.in) + assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("Testing %d", test.in)) + } +} + +func TestNoncePointer(t *testing.T) { + var x nonce + assert.Equal(t, (*[24]byte)(&x), x.pointer()) +} + +func TestNonceFromReader(t *testing.T) { + var x nonce + buf := bytes.NewBufferString("123456789abcdefghijklmno") + err := x.fromReader(buf) + assert.NoError(t, err) + assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x) + buf = bytes.NewBufferString("123456789abcdefghijklmn") + err = x.fromReader(buf) + assert.Error(t, err, "short read of nonce") +} + +func TestNonceFromBuf(t *testing.T) { + var x nonce + buf := []byte("123456789abcdefghijklmnoXXXXXXXX") + x.fromBuf(buf) + assert.Equal(t, nonce{'1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}, x) + buf = []byte("0123456789abcdefghijklmn") + x.fromBuf(buf) + assert.Equal(t, nonce{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n'}, x) + buf = []byte("0123456789abcdefghijklm") + assert.Panics(t, func() { x.fromBuf(buf) }) +} + +func TestNonceIncrement(t *testing.T) { + for _, test := range []struct { + in nonce + out nonce + }{ + { + nonce{0x00}, + nonce{0x01}, + }, + { + nonce{0xFF}, + nonce{0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF}, + nonce{0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + } { + x := test.in + x.increment() + assert.Equal(t, test.out, x) + } +} + +// randomSource can read or write a random sequence +type randomSource struct { + counter int64 + size int64 +} + +func newRandomSource(size int64) *randomSource { + return &randomSource{ + size: size, + } +} + +func (r *randomSource) next() byte { + r.counter++ + return byte(r.counter % 257) +} + +func (r *randomSource) Read(p []byte) (n int, err error) { + for i := range p { + if r.counter >= r.size { + err = io.EOF + break + } + p[i] = r.next() + n++ + } + return n, err +} + +func (r *randomSource) Write(p []byte) (n int, err error) { + for i := range p { + if p[i] != r.next() { + return 0, errors.Errorf("Error in stream at %d", r.counter) + } + } + return len(p), nil +} + +func (r *randomSource) Close() error { return nil } + +// Check interfaces +var ( + _ io.ReadCloser = (*randomSource)(nil) + _ io.WriteCloser = (*randomSource)(nil) +) + +// Test test infrastructure first! +func TestRandomSource(t *testing.T) { + source := newRandomSource(1E8) + sink := newRandomSource(1E8) + n, err := io.Copy(sink, source) + assert.NoError(t, err) + assert.Equal(t, int64(1E8), n) + + source = newRandomSource(1E8) + buf := make([]byte, 16) + _, _ = source.Read(buf) + sink = newRandomSource(1E8) + n, err = io.Copy(sink, source) + assert.Error(t, err, "Error in stream") +} + +type zeroes struct{} + +func (z *zeroes) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = 0 + n++ + } + return n, nil +} + +// Test encrypt decrypt with different buffer sizes +func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) { + c, err := newCipher(0, "") + assert.NoError(t, err) + c.cryptoRand = &zeroes{} // zero out the nonce + buf := make([]byte, bufSize) + source := newRandomSource(copySize) + encrypted, err := c.newEncrypter(source) + assert.NoError(t, err) + decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted)) + assert.NoError(t, err) + sink := newRandomSource(copySize) + n, err := io.CopyBuffer(sink, decrypted, buf) + assert.NoError(t, err) + assert.Equal(t, copySize, n) + blocks := copySize / blockSize + if (copySize % blockSize) != 0 { + blocks++ + } + var expectedNonce = nonce{byte(blocks), byte(blocks >> 8), byte(blocks >> 16), byte(blocks >> 32)} + assert.Equal(t, expectedNonce, encrypted.nonce) + assert.Equal(t, expectedNonce, decrypted.nonce) +} + +func TestEncryptDecrypt1(t *testing.T) { + testEncryptDecrypt(t, 1, 1E7) +} + +func TestEncryptDecrypt32(t *testing.T) { + testEncryptDecrypt(t, 32, 1E8) +} + +func TestEncryptDecrypt4096(t *testing.T) { + testEncryptDecrypt(t, 4096, 1E8) +} + +func TestEncryptDecrypt65536(t *testing.T) { + testEncryptDecrypt(t, 65536, 1E8) +} + +func TestEncryptDecrypt65537(t *testing.T) { + testEncryptDecrypt(t, 65537, 1E8) +} + +var ( + file0 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + } + file1 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x09, 0x5b, 0x44, 0x6c, 0xd6, 0x23, 0x7b, 0xbc, 0xb0, 0x8d, 0x09, 0xfb, 0x52, 0x4c, 0xe5, 0x65, + 0xAA, + } + file16 = []byte{ + 0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0xb9, 0xc4, 0x55, 0x2a, 0x27, 0x10, 0x06, 0x29, 0x18, 0x96, 0x0a, 0x3e, 0x60, 0x8c, 0x29, 0xb9, + 0xaa, 0x8a, 0x5e, 0x1e, 0x16, 0x5b, 0x6d, 0x07, 0x5d, 0xe4, 0xe9, 0xbb, 0x36, 0x7f, 0xd6, 0xd4, + } +) + +func TestEncryptData(t *testing.T) { + for _, test := range []struct { + in []byte + expected []byte + }{ + {[]byte{}, file0}, + {[]byte{1}, file1}, + {[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16}, + } { + c, err := newCipher(0, "") + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + // Check encode works + buf := bytes.NewBuffer(test.in) + encrypted, err := c.EncryptData(buf) + assert.NoError(t, err) + out, err := ioutil.ReadAll(encrypted) + assert.NoError(t, err) + assert.Equal(t, test.expected, out) + + // Check we can decode the data properly too... + buf = bytes.NewBuffer(out) + decrypted, err := c.DecryptData(ioutil.NopCloser(buf)) + assert.NoError(t, err) + out, err = ioutil.ReadAll(decrypted) + assert.NoError(t, err) + assert.Equal(t, test.in, out) + } +} + +func TestNewEncrypter(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + z := &zeroes{} + + fh, err := c.newEncrypter(z) + assert.NoError(t, err) + assert.Equal(t, nonce{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.nonce) + assert.Equal(t, []byte{'R', 'C', 'L', 'O', 'N', 'E', 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, fh.buf[:32]) + + // Test error path + c.cryptoRand = bytes.NewBufferString("123456789abcdefghijklmn") + fh, err = c.newEncrypter(z) + assert.Nil(t, fh) + assert.Error(t, err, "short read of nonce") + +} + +type errorReader struct { + err error +} + +func (er errorReader) Read(p []byte) (n int, err error) { + return 0, er.err +} + +type closeDetector struct { + io.Reader + closed int +} + +func newCloseDetector(in io.Reader) *closeDetector { + return &closeDetector{ + Reader: in, + } +} + +func (c *closeDetector) Close() error { + c.closed++ + return nil +} + +func TestNewDecrypter(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator + + cd := newCloseDetector(bytes.NewBuffer(file0)) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + // check nonce is in place + assert.Equal(t, file0[8:32], fh.nonce[:]) + assert.Equal(t, 0, cd.closed) + + // Test error paths + for i := range file0 { + cd := newCloseDetector(bytes.NewBuffer(file0[:i])) + fh, err = c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, ErrorEncryptedFileTooShort.Error()) + assert.Equal(t, 1, cd.closed) + } + + er := &errorReader{errors.New("potato")} + cd = newCloseDetector(er) + fh, err = c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, "potato") + assert.Equal(t, 1, cd.closed) + + // bad magic + file0copy := make([]byte, len(file0)) + copy(file0copy, file0) + for i := range fileMagic { + file0copy[i] ^= 0x1 + cd := newCloseDetector(bytes.NewBuffer(file0copy)) + fh, err := c.newDecrypter(cd) + assert.Nil(t, fh) + assert.Error(t, err, ErrorEncryptedBadMagic.Error()) + file0copy[i] ^= 0x1 + assert.Equal(t, 1, cd.closed) + } +} + +func TestDecrypterRead(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + + // Test truncating the header + for i := 1; i < blockHeaderSize; i++ { + cd := newCloseDetector(bytes.NewBuffer(file1[:len(file1)-i])) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + _, err = ioutil.ReadAll(fh) + assert.Error(t, err, ErrorEncryptedFileBadHeader.Error()) + assert.Equal(t, 0, cd.closed) + } + + // Test producing an error on the file on Read the underlying file + in1 := bytes.NewBuffer(file1) + in2 := &errorReader{errors.New("potato")} + in := io.MultiReader(in1, in2) + cd := newCloseDetector(in) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + _, err = ioutil.ReadAll(fh) + assert.Error(t, err, "potato") + assert.Equal(t, 0, cd.closed) + + // Test corrupting the input + // shouldn't be able to corrupt any byte without some sort of error + file16copy := make([]byte, len(file16)) + copy(file16copy, file16) + for i := range file16copy { + file16copy[i] ^= 0xFF + fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy))) + if i < fileMagicSize { + assert.Error(t, err, ErrorEncryptedBadMagic.Error()) + assert.Nil(t, fh) + } else { + assert.NoError(t, err) + _, err = ioutil.ReadAll(fh) + assert.Error(t, err, ErrorEncryptedFileBadHeader.Error()) + } + file16copy[i] ^= 0xFF + } +} + +func TestDecrypterClose(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + + cd := newCloseDetector(bytes.NewBuffer(file16)) + fh, err := c.newDecrypter(cd) + assert.NoError(t, err) + assert.Equal(t, 0, cd.closed) + + // close before reading + assert.Equal(t, nil, fh.err) + err = fh.Close() + assert.Equal(t, ErrorFileClosed, fh.err) + assert.Equal(t, 1, cd.closed) + + // double close + err = fh.Close() + assert.Error(t, err, ErrorFileClosed.Error()) + assert.Equal(t, 1, cd.closed) + + // try again reading the file this time + cd = newCloseDetector(bytes.NewBuffer(file1)) + fh, err = c.newDecrypter(cd) + assert.NoError(t, err) + assert.Equal(t, 0, cd.closed) + + // close after reading + out, err := ioutil.ReadAll(fh) + assert.NoError(t, err) + assert.Equal(t, []byte{1}, out) + assert.Equal(t, io.EOF, fh.err) + err = fh.Close() + assert.Equal(t, ErrorFileClosed, fh.err) + assert.Equal(t, 1, cd.closed) +} + +func TestPutGetBlock(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + + block := c.getBlock() + c.putBlock(block) + c.putBlock(block) + + assert.Panics(t, func() { c.putBlock(block[:len(block)-1]) }) +} + +func TestKey(t *testing.T) { + c, err := newCipher(0, "") + assert.NoError(t, err) + + // Check zero keys OK + assert.Equal(t, [32]byte{}, c.dataKey) + assert.Equal(t, [32]byte{}, c.nameKey) + assert.Equal(t, [16]byte{}, c.nameTweak) + + require.NoError(t, c.Key("potato")) + assert.Equal(t, [32]byte{0x74, 0x55, 0xC7, 0x1A, 0xB1, 0x7C, 0x86, 0x5B, 0x84, 0x71, 0xF4, 0x7B, 0x79, 0xAC, 0xB0, 0x7E, 0xB3, 0x1D, 0x56, 0x78, 0xB8, 0x0C, 0x7E, 0x2E, 0xAF, 0x4F, 0xC8, 0x06, 0x6A, 0x9E, 0xE4, 0x68}, c.dataKey) + assert.Equal(t, [32]byte{0x76, 0x5D, 0xA2, 0x7A, 0xB1, 0x5D, 0x77, 0xF9, 0x57, 0x96, 0x71, 0x1F, 0x7B, 0x93, 0xAD, 0x63, 0xBB, 0xB4, 0x84, 0x07, 0x2E, 0x71, 0x80, 0xA8, 0xD1, 0x7A, 0x9B, 0xBE, 0xC1, 0x42, 0x70, 0xD0}, c.nameKey) + assert.Equal(t, [16]byte{0xC1, 0x8D, 0x59, 0x32, 0xF5, 0x5B, 0x28, 0x28, 0xC5, 0xE1, 0xE8, 0x72, 0x15, 0x52, 0x03, 0x10}, c.nameTweak) + + require.NoError(t, c.Key("Potato")) + assert.Equal(t, [32]byte{0xAE, 0xEA, 0x6A, 0xD3, 0x47, 0xDF, 0x75, 0xB9, 0x63, 0xCE, 0x12, 0xF5, 0x76, 0x23, 0xE9, 0x46, 0xD4, 0x2E, 0xD8, 0xBF, 0x3E, 0x92, 0x8B, 0x39, 0x24, 0x37, 0x94, 0x13, 0x3E, 0x5E, 0xF7, 0x5E}, c.dataKey) + assert.Equal(t, [32]byte{0x54, 0xF7, 0x02, 0x6E, 0x8A, 0xFC, 0x56, 0x0A, 0x86, 0x63, 0x6A, 0xAB, 0x2C, 0x9C, 0x51, 0x62, 0xE5, 0x1A, 0x12, 0x23, 0x51, 0x83, 0x6E, 0xAF, 0x50, 0x42, 0x0F, 0x98, 0x1C, 0x86, 0x0A, 0x19}, c.nameKey) + assert.Equal(t, [16]byte{0xF8, 0xC1, 0xB6, 0x27, 0x2D, 0x52, 0x9B, 0x4A, 0x8F, 0xDA, 0xEB, 0x42, 0x4A, 0x28, 0xDD, 0xF3}, c.nameTweak) + + require.NoError(t, c.Key("")) + assert.Equal(t, [32]byte{}, c.dataKey) + assert.Equal(t, [32]byte{}, c.nameKey) + assert.Equal(t, [16]byte{}, c.nameTweak) +} diff --git a/crypt/crypt.go b/crypt/crypt.go new file mode 100644 index 000000000..26f840eb9 --- /dev/null +++ b/crypt/crypt.go @@ -0,0 +1,445 @@ +// Package crypt provides wrappers for Fs and Object which implement encryption +package crypt + +import ( + "fmt" + "io" + "path" + "strings" + "sync" + + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "crypt", + Description: "Encrypt/Decrypt a remote", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "remote", + Help: "Remote to encrypt/decrypt.", + }, { + Name: "flatten", + Help: "Flatten the directory structure - more secure, less useful - see docs for tradeoffs.", + Examples: []fs.OptionExample{ + { + Value: "0", + Help: "Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure.", + }, { + Value: "1", + Help: "Spread files over 1 directory good for <10,000 files.", + }, { + Value: "2", + Help: "Spread files over 32 directories good for <320,000 files.", + }, { + Value: "3", + Help: "Spread files over 1024 directories good for <10,000,000 files.", + }, { + Value: "4", + Help: "Spread files over 32,768 directories good for <320,000,000 files.", + }, { + Value: "5", + Help: "Spread files over 1,048,576 levels good for <10,000,000,000 files.", + }, + }, + }, { + Name: "password", + Help: "Password or pass phrase for encryption.", + IsPassword: true, + }}, + }) +} + +// NewFs contstructs an Fs from the path, container:path +func NewFs(name, rpath string) (fs.Fs, error) { + flatten := fs.ConfigFile.MustInt(name, "flatten", 0) + password := fs.ConfigFile.MustValue(name, "password", "") + if password == "" { + return nil, errors.New("password not set in config file") + } + password, err := fs.Reveal(password) + if err != nil { + return nil, errors.Wrap(err, "failed to decrypt password") + } + cipher, err := newCipher(flatten, password) + if err != nil { + return nil, errors.Wrap(err, "failed to make cipher") + } + remote := fs.ConfigFile.MustValue(name, "remote") + remotePath := path.Join(remote, cipher.EncryptName(rpath)) + wrappedFs, err := fs.NewFs(remotePath) + if err != fs.ErrorIsFile && err != nil { + return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remotePath) + } + f := &Fs{ + Fs: wrappedFs, + cipher: cipher, + flatten: flatten, + } + return f, err +} + +// Fs represents a wrapped fs.Fs +type Fs struct { + fs.Fs + cipher Cipher + flatten int +} + +// String returns a description of the FS +func (f *Fs) String() string { + return fmt.Sprintf("%s with cipher", f.Fs.String()) +} + +// List the Fs into a channel +func (f *Fs) List(opts fs.ListOpts, dir string) { + f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptName(dir)) +} + +// NewObject finds the Object at remote. +func (f *Fs) NewObject(remote string) (fs.Object, error) { + o, err := f.Fs.NewObject(f.cipher.EncryptName(remote)) + if err != nil { + return nil, err + } + return f.newObject(o), nil +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { + wrappedIn, err := f.cipher.EncryptData(in) + if err != nil { + return nil, err + } + o, err := f.Fs.Put(wrappedIn, f.newObjectInfo(src)) + if err != nil { + return nil, err + } + return f.newObject(o), nil +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() fs.HashSet { + return fs.HashSet(fs.HashNone) +} + +// Purge all files in the root and the root directory +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge() error { + do, ok := f.Fs.(fs.Purger) + if !ok { + return fs.ErrorCantPurge + } + return do.Purge() +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + do, ok := f.Fs.(fs.Copier) + if !ok { + return nil, fs.ErrorCantCopy + } + o, ok := src.(*Object) + if !ok { + return nil, fs.ErrorCantCopy + } + oResult, err := do.Copy(o.Object, f.cipher.EncryptName(remote)) + if err != nil { + return nil, err + } + return f.newObject(oResult), nil +} + +// Move src to this remote using server side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + do, ok := f.Fs.(fs.Mover) + if !ok { + return nil, fs.ErrorCantMove + } + o, ok := src.(*Object) + if !ok { + return nil, fs.ErrorCantCopy + } + oResult, err := do.Move(o.Object, f.cipher.EncryptName(remote)) + if err != nil { + return nil, err + } + return f.newObject(oResult), nil +} + +// UnWrap returns the Fs that this Fs is wrapping +func (f *Fs) UnWrap() fs.Fs { + return f.Fs +} + +// Object describes a wrapped for being read from the Fs +// +// This decrypts the remote name and decrypts the data +type Object struct { + fs.Object + f *Fs +} + +func (f *Fs) newObject(o fs.Object) *Object { + return &Object{ + Object: o, + f: f, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (o *Object) Fs() fs.Info { + return o.f +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.Remote() +} + +// Remote returns the remote path +func (o *Object) Remote() string { + remote := o.Object.Remote() + decryptedName, err := o.f.cipher.DecryptName(remote) + if err != nil { + fs.Debug(remote, "Undecryptable file name: %v", err) + return remote + } + return decryptedName +} + +// Size returns the size of the file +func (o *Object) Size() int64 { + size, err := o.f.cipher.DecryptedSize(o.Object.Size()) + if err != nil { + fs.Debug(o, "Bad size for decrypt: %v", err) + } + return size +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *Object) Hash(hash fs.HashType) (string, error) { + return "", nil +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o *Object) Open() (io.ReadCloser, error) { + in, err := o.Object.Open() + if err != nil { + return in, err + } + return o.f.cipher.DecryptData(in) +} + +// Update in to the object with the modTime given of the given size +func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error { + wrappedIn, err := o.f.cipher.EncryptData(in) + if err != nil { + return err + } + return o.Object.Update(wrappedIn, o.f.newObjectInfo(src)) +} + +// newDir returns a dir with the Name decrypted +func (f *Fs) newDir(dir *fs.Dir) *fs.Dir { + new := *dir + remote := dir.Name + decryptedRemote, err := f.cipher.DecryptName(remote) + if err != nil { + fs.Debug(remote, "Undecryptable dir name: %v", err) + } else { + new.Name = decryptedRemote + } + return &new +} + +// ObjectInfo describes a wrapped fs.ObjectInfo for being the source +// +// This encrypts the remote name and adjusts the size +type ObjectInfo struct { + fs.ObjectInfo + f *Fs +} + +func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo { + return &ObjectInfo{ + ObjectInfo: src, + f: f, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (o *ObjectInfo) Fs() fs.Info { + return o.f +} + +// Remote returns the remote path +func (o *ObjectInfo) Remote() string { + return o.f.cipher.EncryptName(o.ObjectInfo.Remote()) +} + +// Size returns the size of the file +func (o *ObjectInfo) Size() int64 { + return o.f.cipher.EncryptedSize(o.ObjectInfo.Size()) +} + +// ListOpts wraps a listopts decrypting the directory listing and +// replacing the Objects +type ListOpts struct { + fs.ListOpts + f *Fs + dir string // dir we are listing + mu sync.Mutex // to protect dirs + dirs map[string]struct{} // keep track of synthetic directory objects added +} + +// Make a ListOpts wrapper +func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts { + if dir != "" { + dir += "/" + } + return &ListOpts{ + ListOpts: lo, + f: f, + dir: dir, + dirs: make(map[string]struct{}), + } + +} + +// Level gets the recursion level for this listing. +// +// Fses may ignore this, but should implement it for improved efficiency if possible. +// +// Level 1 means list just the contents of the directory +// +// Each returned item must have less than level `/`s in. +func (lo *ListOpts) Level() int { + // If flattened recurse fully + if lo.f.flatten > 0 { + return fs.MaxLevel + } + return lo.ListOpts.Level() +} + +// addSyntheticDirs makes up directory objects for the path passed in +func (lo *ListOpts) addSyntheticDirs(path string) { + lo.mu.Lock() + defer lo.mu.Unlock() + for { + i := strings.LastIndexByte(path, '/') + if i < 0 { + break + } + path = path[:i] + if path == "" { + break + } + if _, found := lo.dirs[path]; found { + break + } + slashes := strings.Count(path, "/") + if slashes < lo.ListOpts.Level() { + lo.ListOpts.AddDir(&fs.Dir{Name: path}) + } + lo.dirs[path] = struct{}{} + } +} + +// Add an object to the output. +// If the function returns true, the operation has been aborted. +// Multiple goroutines can safely add objects concurrently. +func (lo *ListOpts) Add(obj fs.Object) (abort bool) { + remote := obj.Remote() + decryptedRemote, err := lo.f.cipher.DecryptName(remote) + if err != nil { + fs.Debug(remote, "Skipping undecryptable file name: %v", err) + return lo.ListOpts.IsFinished() + } + // If flattened add synthetic directories + if lo.f.flatten > 0 { + lo.addSyntheticDirs(decryptedRemote) + slashes := strings.Count(decryptedRemote, "/") + if slashes >= lo.ListOpts.Level() { + return lo.ListOpts.IsFinished() + } + } + return lo.ListOpts.Add(lo.f.newObject(obj)) +} + +// AddDir adds a directory to the output. +// If the function returns true, the operation has been aborted. +// Multiple goroutines can safely add objects concurrently. +func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) { + // If flattened we don't add any directories from the underlying remote + if lo.f.flatten > 0 { + return lo.ListOpts.IsFinished() + } + remote := dir.Name + _, err := lo.f.cipher.DecryptName(remote) + if err != nil { + fs.Debug(remote, "Skipping undecryptable dir name: %v", err) + return lo.ListOpts.IsFinished() + } + return lo.ListOpts.AddDir(lo.f.newDir(dir)) +} + +// IncludeDirectory returns whether this directory should be +// included in the listing (and recursed into or not). +func (lo *ListOpts) IncludeDirectory(remote string) bool { + // If flattened we look in all directories + if lo.f.flatten > 0 { + return true + } + decryptedRemote, err := lo.f.cipher.DecryptName(remote) + if err != nil { + fs.Debug(remote, "Not including undecryptable directory name: %v", err) + return false + } + return lo.ListOpts.IncludeDirectory(decryptedRemote) +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + // _ fs.DirMover = (*Fs)(nil) + // _ fs.PutUncheckeder = (*Fs)(nil) + _ fs.UnWrapper = (*Fs)(nil) + _ fs.ObjectInfo = (*ObjectInfo)(nil) + _ fs.Object = (*Object)(nil) + _ fs.ListOpts = (*ListOpts)(nil) +) diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go new file mode 100644 index 000000000..8f73f5543 --- /dev/null +++ b/crypt/crypt_test.go @@ -0,0 +1,59 @@ +// Test Crypt filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package crypt_test + +import ( + "testing" + + "github.com/ncw/rclone/crypt" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + _ "github.com/ncw/rclone/local" +) + +func init() { + fstests.NilObject = fs.Object((*crypt.Object)(nil)) + fstests.RemoteName = "TestCrypt:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } +func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/crypt/pkcs7/pkcs7.go b/crypt/pkcs7/pkcs7.go new file mode 100644 index 000000000..e6d9d0fd9 --- /dev/null +++ b/crypt/pkcs7/pkcs7.go @@ -0,0 +1,63 @@ +// Package pkcs7 implements PKCS#7 padding +// +// This is a standard way of encoding variable length buffers into +// buffers which are a multiple of an underlying crypto block size. +package pkcs7 + +import "github.com/pkg/errors" + +// Errors Unpad can return +var ( + ErrorPaddingNotFound = errors.New("Bad PKCS#7 padding - not padded") + ErrorPaddingNotAMultiple = errors.New("Bad PKCS#7 padding - not a multiple of blocksize") + ErrorPaddingTooLong = errors.New("Bad PKCS#7 padding - too long") + ErrorPaddingTooShort = errors.New("Bad PKCS#7 padding - too short") + ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same") +) + +// Pad buf using PKCS#7 to a multiple of n. +// +// Appends the padding to buf - make a copy of it first if you don't +// want it modified. +func Pad(n int, buf []byte) []byte { + if n <= 1 || n >= 256 { + panic("bad multiple") + } + length := len(buf) + padding := n - (length % n) + for i := 0; i < padding; i++ { + buf = append(buf, byte(padding)) + } + if (len(buf) % n) != 0 { + panic("padding failed") + } + return buf +} + +// Unpad buf using PKCS#7 from a multiple of n returning a slice of +// buf or an error if malformed. +func Unpad(n int, buf []byte) ([]byte, error) { + if n <= 1 || n >= 256 { + panic("bad multiple") + } + length := len(buf) + if length == 0 { + return nil, ErrorPaddingNotFound + } + if (length % n) != 0 { + return nil, ErrorPaddingNotAMultiple + } + padding := int(buf[length-1]) + if padding > n { + return nil, ErrorPaddingTooLong + } + if padding == 0 { + return nil, ErrorPaddingTooShort + } + for i := 0; i < padding; i++ { + if buf[length-1-i] != byte(padding) { + return nil, ErrorPaddingNotAllTheSame + } + } + return buf[:length-padding], nil +} diff --git a/crypt/pkcs7/pkcs7_test.go b/crypt/pkcs7/pkcs7_test.go new file mode 100644 index 000000000..2264c7fd3 --- /dev/null +++ b/crypt/pkcs7/pkcs7_test.go @@ -0,0 +1,73 @@ +package pkcs7 + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPad(t *testing.T) { + for _, test := range []struct { + n int + in string + expected string + }{ + {8, "", "\x08\x08\x08\x08\x08\x08\x08\x08"}, + {8, "1", "1\x07\x07\x07\x07\x07\x07\x07"}, + {8, "12", "12\x06\x06\x06\x06\x06\x06"}, + {8, "123", "123\x05\x05\x05\x05\x05"}, + {8, "1234", "1234\x04\x04\x04\x04"}, + {8, "12345", "12345\x03\x03\x03"}, + {8, "123456", "123456\x02\x02"}, + {8, "1234567", "1234567\x01"}, + {8, "abcdefgh", "abcdefgh\x08\x08\x08\x08\x08\x08\x08\x08"}, + {8, "abcdefgh1", "abcdefgh1\x07\x07\x07\x07\x07\x07\x07"}, + {8, "abcdefgh12", "abcdefgh12\x06\x06\x06\x06\x06\x06"}, + {8, "abcdefgh123", "abcdefgh123\x05\x05\x05\x05\x05"}, + {8, "abcdefgh1234", "abcdefgh1234\x04\x04\x04\x04"}, + {8, "abcdefgh12345", "abcdefgh12345\x03\x03\x03"}, + {8, "abcdefgh123456", "abcdefgh123456\x02\x02"}, + {8, "abcdefgh1234567", "abcdefgh1234567\x01"}, + {8, "abcdefgh12345678", "abcdefgh12345678\x08\x08\x08\x08\x08\x08\x08\x08"}, + {16, "", "\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10"}, + {16, "a", "a\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f\x0f"}, + } { + actual := Pad(test.n, []byte(test.in)) + assert.Equal(t, test.expected, string(actual), fmt.Sprintf("Pad %d %q", test.n, test.in)) + recovered, err := Unpad(test.n, actual) + assert.NoError(t, err) + assert.Equal(t, []byte(test.in), recovered, fmt.Sprintf("Unpad %d %q", test.n, test.in)) + } + assert.Panics(t, func() { Pad(1, []byte("")) }, "bad multiple") + assert.Panics(t, func() { Pad(256, []byte("")) }, "bad multiple") +} + +func TestUnpad(t *testing.T) { + // We've tested the OK decoding in TestPad, now test the error cases + for _, test := range []struct { + n int + in string + err error + }{ + {8, "", ErrorPaddingNotFound}, + {8, "1", ErrorPaddingNotAMultiple}, + {8, "12", ErrorPaddingNotAMultiple}, + {8, "123", ErrorPaddingNotAMultiple}, + {8, "1234", ErrorPaddingNotAMultiple}, + {8, "12345", ErrorPaddingNotAMultiple}, + {8, "123456", ErrorPaddingNotAMultiple}, + {8, "1234567", ErrorPaddingNotAMultiple}, + {8, "1234567\xFF", ErrorPaddingTooLong}, + {8, "1234567\x09", ErrorPaddingTooLong}, + {8, "1234567\x00", ErrorPaddingTooShort}, + {8, "123456\x01\x02", ErrorPaddingNotAllTheSame}, + {8, "\x07\x08\x08\x08\x08\x08\x08\x08", ErrorPaddingNotAllTheSame}, + } { + result, actualErr := Unpad(test.n, []byte(test.in)) + assert.Equal(t, test.err, actualErr, fmt.Sprintf("Unpad %d %q", test.n, test.in)) + assert.Equal(t, result, []byte(nil)) + } + assert.Panics(t, func() { _, _ = Unpad(1, []byte("")) }, "bad multiple") + assert.Panics(t, func() { _, _ = Unpad(256, []byte("")) }, "bad multiple") +} diff --git a/docs/content/crypt.md b/docs/content/crypt.md new file mode 100644 index 000000000..5ac4ffb26 --- /dev/null +++ b/docs/content/crypt.md @@ -0,0 +1,288 @@ +--- +title: "Crypt" +description: "Encryption overlay remote" +date: "2016-07-28" +--- + +Crypt +---------------------------------------- + +The `crypt` remote encrypts and decrypts another remote. + +To use it first set up the underlying remote following the config +instructions for that remote. You can also use a local pathname +instead of a remote which will encrypt and decrypt from that directory +which might be useful for encrypting onto a USB stick for example. + +First check your chosen remote is working - we'll call it +`remote:path` in these docs. Note that anything inside `remote:path` +will be encrypted and anything outside won't. This means that if you +are using a bucket based remote (eg S3, B2, swift) then you should +probably put the bucket in the remote `s3:bucket`. If you just use +`s3:` then rclone will make encrypted bucket names too which may or +may not be what you want. + +Now configure `crypt` using `rclone config`. We will call this one +`secret` to differentiate it from the `remote`. + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> secret +Type of storage to configure. +Choose a number from below, or type in your own value + 1 / Amazon Drive + \ "amazon cloud drive" + 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + \ "s3" + 3 / Backblaze B2 + \ "b2" + 4 / Dropbox + \ "dropbox" + 5 / Encrypt/Decrypt a remote + \ "crypt" + 6 / Google Cloud Storage (this is not Google Drive) + \ "google cloud storage" + 7 / Google Drive + \ "drive" + 8 / Hubic + \ "hubic" + 9 / Local Disk + \ "local" +10 / Microsoft OneDrive + \ "onedrive" +11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) + \ "swift" +12 / Yandex Disk + \ "yandex" +Storage> 5 +Remote to encrypt/decrypt. +remote> remote:path +Flatten the directory structure - more secure, less useful - see docs for tradeoffs. +Choose a number from below, or type in your own value + 1 / Don't flatten files (default) - good for unlimited files, but doesn't hide directory structure. + \ "0" + 2 / Spread files over 1 directory good for <10,000 files. + \ "1" + 3 / Spread files over 32 directories good for <320,000 files. + \ "2" + 4 / Spread files over 1024 directories good for <10,000,000 files. + \ "3" + 5 / Spread files over 32,768 directories good for <320,000,000 files. + \ "4" + 6 / Spread files over 1,048,576 levels good for <10,000,000,000 files. + \ "5" +flatten> 1 +Password or pass phrase for encryption. +Enter the password: +password: +Confirm the password: +password: +Remote config +-------------------- +[secret] +remote = remote:path +flatten = 0 +password = 0_gtCJ422bzwAWP0UN2lggrjhA-sSg +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +**Important** The password is stored in the config file is lightly +obscured so it isn't immediately obvious what it is. It is in no way +secure unless you use config file encryption. + +A long passphrase is recommended, or you can use a random one. Note +that if you reconfigure rclone with the same password/passphrase +elsewhere it will be compatible - all the secrets used are derived +from that one password/passphrase. + +Note that rclone does not encrypt + * file length - this can be calcuated within 16 bytes + * modification time - used for syncing + +## Example ## + +To test I made a little directory of files + +``` +plaintext/ +├── file0.txt +├── file1.txt +└── subdir + ├── file2.txt + ├── file3.txt + └── subsubdir + └── file4.txt +``` + +Copy these to the remote and list them back + +``` +$ rclone -q copy plaintext secret: +$ rclone -q ls secret: + 7 file1.txt + 6 file0.txt + 8 subdir/file2.txt + 10 subdir/subsubdir/file4.txt + 9 subdir/file3.txt +``` + +Now see what that looked like when encrypted + +``` +$ rclone -q ls remote:path + 55 hagjclgavj2mbiqm6u6cnjjqcg + 54 v05749mltvv1tf4onltun46gls + 57 86vhrsv86mpbtd3a0akjuqslj8/dlj7fkq4kdq72emafg7a7s41uo + 58 86vhrsv86mpbtd3a0akjuqslj8/7uu829995du6o42n32otfhjqp4/b9pausrfansjth5ob3jkdqd4lc + 56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps +``` + +Note that this retains the directory structure which means you can do this + +``` +$ rclone -q ls secret:subdir + 8 file2.txt + 9 file3.txt + 10 subsubdir/file4.txt +``` + +If you use the flattened flag then the listing will look and that last command will not work. + +``` +$ rclone -q ls remote:path + 56 t/tsdtcpdu6g9dpamn6poqc248tll9dj5ok78a363etmq8ushr821g + 57 g/gsrp2g0u85pgsi6kso74bjsrsafe11odpfln8qqpj6n9p20of0a0 + 55 h/hagjclgavj2mbiqm6u6cnjjqcg + 58 4/4jsbao3dhi0jfoubt2oo493pbqmsshn92q01ddu7dg6428rlluhg + 54 v/v05749mltvv1tf4onltun46gls +``` + +### Flattened vs non-Flattened ### + +Pros and cons of each + +Flattened + * hides directory structures + * identical file names won't have identical encrypted names + * can't use a sub path + * doesn't work: `rclone copy crypt:sub/dir /tmp/recovered` + * use: `rclone copy --include "/sub/dir/**" crypt: /tmp/recovered` + * will always have to recurse through the entire directory structure + * can't copy a single file directly + * doesn't work: `rclone copy crypt:path/to/file /tmp/recovered` + * use: `rclone copy --include "/path/to/file" crypt: /tmp/recovered` + +Normal + * can use sub paths and copy single files + * directory structure visibile + * identical files names will have identical uploaded names + * can use shortcuts to shorten the directory recursion + +You can swap between flattened levels without re-uploading your files. + +## File formats ## + +### File encryption ### + +Files are encrypted 1:1 source file to destination object. The file +has a header and is divided into chunks. + +#### Header #### + + * 8 bytes magic string `RCLONE\x00\x00` + * 24 bytes Nonce (IV) + +The initial nonce is generated from the operating systems crypto +strong random number genrator. The nonce is incremented for each +chunk read making sure each nonce is unique for each block written. +The chance of a nonce being re-used is miniscule. If you wrote an +exabyte of data (10¹⁸ bytes) you would have a probability of +approximately 2×10⁻³² of re-using a nonce. + +#### Chunk #### + +Each chunk will contain 64kB of data, except for the last one which +may have less data. The data chunk is in standard NACL secretbox +format. Secretbox uses XSalsa20 and Poly1305 to encrypt and +authenticate messages. + +Each chunk contains: + + * 16 Bytes of Poly1305 authenticator + * 1 - 65536 bytes XSalsa20 encrypted data + +64k chunk size was chosen as the best performing chunk size (the +authenticator takes too much time below this and the performance drops +off due to cache effects above this). Note that these chunks are +buffered in memory so they can't be too big. + +This uses a 32 byte (256 bit key) key derived from the user password. + +#### Examples #### + +1 byte file will encrypt to + + * 32 bytes header + * 17 bytes data chunk + +49 bytes total + +1MB (1048576 bytes) file will encrypt to + + * 32 bytes header + * 16 chunks of 65568 bytes + +1049120 bytes total (a 0.05% overhead). This is the overhead for big +files. + +### Name encryption ### + +File names are encrypted by crypt. These are either encrypted segment +by segment - the path is broken up into `/` separated strings and +these are encrypted individually, or if working in flattened mode the +whole path is encrypted `/`s and all. + +First file names are padded using using PKCS#7 to a multiple of 16 +bytes before encryption. + +They are then encrypted with EME using AES with 256 bit key. EME +(ECB-Mix-ECB) is a wide-block encryption mode presented in the 2003 +paper "A Parallelizable Enciphering Mode" by Halevi and Rogaway. + +This makes for determinstic encryption which is what we want - the +same filename must encrypt to the same thing. + +This means that + + * filenames with the same name will encrypt the same + * (though we can use directory flattening to avoid this if required) + * filenames which start the same won't have a common prefix + +This uses a 32 byte key (256 bits) and a 16 byte (128 bits) IV both of +which are derived from the user password. + +After encryption they are written out using a modified version of +standard `base32` encoding as described in RFC4648. The standard +encoding is modified in two ways: + + * it becomes lower case (no-one likes upper case filenames!) + * we strip the padding character `=` + +`base32` is used rather than the more efficient `base64` so rclone can be +used on case insensitive remotes (eg Windows, Amazon Drive). + +### Key derivation ### + +Rclone uses `scrypt` with parameters `N=16384, r=8, p=1` with a fixed +salt to derive the 32+32+16 = 80 bytes of key material required. + +`scrypt` makes it impractical to mount a dictionary attack on rclone +encrypted data. diff --git a/fs/all/all.go b/fs/all/all.go index d68efd6cf..06016df2b 100644 --- a/fs/all/all.go +++ b/fs/all/all.go @@ -4,6 +4,7 @@ import ( // Active file systems _ "github.com/ncw/rclone/amazonclouddrive" _ "github.com/ncw/rclone/b2" + _ "github.com/ncw/rclone/crypt" _ "github.com/ncw/rclone/drive" _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index eb167e7c1..52e073322 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -8,6 +8,7 @@ package fstests import ( "bytes" "flag" + "fmt" "io" "os" "path" @@ -86,7 +87,7 @@ func TestInit(t *testing.T) { t.Logf("Didn't find %q in config file - skipping tests", RemoteName) return } - require.NoError(t, err) + require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err)) fstest.TestMkdir(t, remote) } @@ -215,7 +216,7 @@ again: tries++ goto again } - require.NoError(t, err, "Put error") + require.NoError(t, err, fmt.Sprintf("Put error: %v", err)) } file.Hashes = hash.Sums() file.Check(t, obj, remote.Precision()) @@ -335,7 +336,10 @@ func TestFsCopy(t *testing.T) { // do the copy src := findObject(t, file1.Path) dst, err := remote.(fs.Copier).Copy(src, file1Copy.Path) - require.NoError(t, err) + if err == fs.ErrorCantCopy { + t.Skip("FS can't copy") + } + require.NoError(t, err, fmt.Sprintf("Error: %#v", err)) // check file exists in new listing fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file1Copy}) @@ -365,6 +369,9 @@ func TestFsMove(t *testing.T) { // do the move src := findObject(t, file1.Path) dst, err := remote.(fs.Mover).Move(src, file1Move.Path) + if err == fs.ErrorCantMove { + t.Skip("FS can't move") + } require.NoError(t, err) // check file exists in new listing @@ -521,7 +528,7 @@ func TestObjectOpen(t *testing.T) { require.NoError(t, err) hasher := fs.NewMultiHasher() n, err := io.Copy(hasher, in) - require.NoError(t, err) + require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err)) require.Equal(t, file1.Size, n, "Read wrong number of bytes") err = in.Close() require.NoError(t, err) diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index 040e94ed8..366fe56cd 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -61,7 +61,8 @@ import ( "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fstest/fstests" "github.com/ncw/rclone/{{ .FsName }}" -) +{{ if eq .FsName "crypt" }} _ "github.com/ncw/rclone/local" +{{end}}) func init() { fstests.NilObject = fs.Object((*{{ .FsName }}.Object)(nil)) @@ -135,5 +136,6 @@ func main() { generateTestProgram(t, fns, "Hubic") generateTestProgram(t, fns, "B2") generateTestProgram(t, fns, "Yandex") + generateTestProgram(t, fns, "Crypt") log.Printf("Done") }