github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-29 07:55:12 +01:00
Code Releases Activity

crypt: speed up repeated seeking - fixes #804

Browse Source
This commit is contained in:
Nick Craig-Wood 2016-10-20 17:47:33 +01:00
parent de73063977
commit eba0a3633b
4 changed files with 196 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cmd/mount/read.go
View File

@ -41,17 +41,28 @@ var _ fusefs.HandleReader = (*ReadFileHandle)(nil)
// seek to a new offset
func (fh *ReadFileHandle) seek(offset int64) error {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset)
r, err := fh.o.Open(&fs.SeekOption{Offset: offset})
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err)
return err
// Can we seek it directly?
if do, ok := fh.r.(io.Seeker); ok {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d (io.Seeker)", fh.offset, offset)
_, err := do.Seek(offset, io.SeekStart)
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read io.Seeker failed: %v", err)
return err
}
} else {
fs.Debug(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset)
// if not re-open with a seek
r, err := fh.o.Open(&fs.SeekOption{Offset: offset})
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek failed: %v", err)
return err
}
err = fh.r.Close()
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err)
}
fh.r = r
}
err = fh.r.Close()
if err != nil {
fs.Debug(fh.o, "ReadFileHandle.Read seek close old failed: %v", err)
}
fh.r = r
fh.offset = offset
return nil
}

133
crypt/cipher.go
View File

@ -8,6 +8,7 @@ import (
"encoding/base32"
"fmt"
"io"
"io/ioutil"
"strings"
"sync"
"unicode/utf8"
@ -21,7 +22,7 @@ import (
"github.com/rfjakob/eme"
)
// Constancs
// Constants
const (
nameCipherBlockSize = aes.BlockSize
fileMagic = "RCLONE\x00\x00"
@ -55,6 +56,16 @@ var (
fileMagicBytes = []byte(fileMagic)
)
// ReadSeekCloser is the interface of the read handles
type ReadSeekCloser interface {
io.Reader
io.Seeker
io.Closer
}
// OpenAtOffset opens the file handle at the offset given
type OpenAtOffset func(offset int64) (io.ReadCloser, error)
// Cipher is used to swap out the encryption implementations
type Cipher interface {
// EncryptFileName encrypts a file path
@ -69,6 +80,8 @@ type Cipher interface {
EncryptData(io.Reader) (io.Reader, error)
// DecryptData
DecryptData(io.ReadCloser) (io.ReadCloser, error)
// DecryptDataSeek decrypt at a given position
DecryptDataSeek(open OpenAtOffset, offset int64) (ReadSeekCloser, error)
// EncryptedSize calculates the size of the data when encrypted
EncryptedSize(int64) int64
// DecryptedSize calculates the size of the data when decrypted
@ -476,14 +489,16 @@ func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
// decrypter decrypts an io.ReaderCloser on the fly
type decrypter struct {
rc io.ReadCloser
nonce nonce
c *cipher
buf []byte
readBuf []byte
bufIndex int
bufSize int
err error
rc io.ReadCloser
nonce nonce
initialNonce nonce
c *cipher
buf []byte
readBuf []byte
bufIndex int
bufSize int
err error
open OpenAtOffset
}
// newDecrypter creates a new file handle decrypting on the fly
@ -509,6 +524,30 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
}
// retreive the nonce
fh.nonce.fromBuf(readBuf[fileMagicSize:])
fh.initialNonce = fh.nonce
return fh, nil
}
// newDecrypterSeek creates a new file handle decrypting on the fly
func (c *cipher) newDecrypterSeek(open OpenAtOffset, offset int64) (fh *decrypter, err error) {
// Open initially with no seek
rc, err := open(0)
if err != nil {
return nil, err
}
// Open the stream which fills in the nonce
fh, err = c.newDecrypter(rc)
if err != nil {
return nil, err
}
fh.open = open // will be called by fh.Seek
if offset != 0 {
_, err = fh.Seek(offset, io.SeekStart)
if err != nil {
_ = fh.Close()
return nil, err
}
}
return fh, nil
}
@ -549,15 +588,60 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
return n, nil
}
// seek the decryption forwards the amount given
//
// returns an offset for the underlying rc to be seeked and the number
// of bytes to be discarded
func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) {
blocks, discard := offset/blockDataSize, offset%blockDataSize
underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
fh.nonce.add(uint64(blocks))
return
// Seek as per io.Seeker
func (fh *decrypter) Seek(offset int64, whence int) (int64, error) {
if fh.open == nil {
return 0, fh.finish(errors.New("can't seek - not initialised with newDecrypterSeek"))
}
if whence != io.SeekStart {
return 0, fh.finish(errors.New("can only seek from the start"))
}
// Reset error or return it if not EOF
if fh.err == io.EOF {
fh.err = nil
} else if fh.err != nil {
return 0, fh.err
}
// Can we seek it directly?
if do, ok := fh.rc.(io.Seeker); ok {
_, err := do.Seek(offset, io.SeekStart)
if err != nil {
return 0, fh.finish(err)
}
} else {
// if not reopen with seek
_ = fh.rc.Close() // close underlying file
fh.rc = nil
// blocks we need to seek, plus bytes we need to discard
blocks, discard := offset/blockDataSize, offset%blockDataSize
// Offset in underlying stream we need to seek
underlyingOffset := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
// Move the nonce on the correct number of blocks from the start
fh.nonce = fh.initialNonce
fh.nonce.add(uint64(blocks))
// Re-open the underlying object with the offset given
rc, err := fh.open(underlyingOffset)
if err != nil {
return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset"))
}
// Set the file handle
fh.rc = rc
// Discard excess bytes
_, err = io.CopyN(ioutil.Discard, fh, discard)
if err != nil {
return 0, fh.finish(err)
}
}
return offset, nil
}
// finish sets the final error and tidies up
@ -604,6 +688,19 @@ func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
return out, nil
}
// DecryptDataSeek decrypts the data stream from offset
//
// The open function must return a ReadCloser opened to the offset supplied
//
// You must use this form of DecryptData if you might want to Seek the file handle
func (c *cipher) DecryptDataSeek(open OpenAtOffset, offset int64) (ReadSeekCloser, error) {
out, err := c.newDecrypterSeek(open, offset)
if err != nil {
return nil, err
}
return out, nil
}
// EncryptedSize calculates the size of the data when encrypted
func (c *cipher) EncryptedSize(size int64) int64 {
blocks, residue := size/blockDataSize, size%blockDataSize

51
crypt/cipher_test.go
View File

@ -854,6 +854,57 @@ func TestNewDecrypter(t *testing.T) {
}
}
func TestNewDecrypterSeek(t *testing.T) {
c, err := newCipher(NameEncryptionStandard, "", "")
assert.NoError(t, err)
c.cryptoRand = &zeroes{} // nodge the crypto rand generator
// Make random data
const dataSize = 150000
plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
assert.NoError(t, err)
// Encrypt the data
buf := bytes.NewBuffer(plaintext)
encrypted, err := c.EncryptData(buf)
assert.NoError(t, err)
ciphertext, err := ioutil.ReadAll(encrypted)
assert.NoError(t, err)
trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049,
4095, 4096, 4097, 8191, 8192, 8193, 16383, 16384, 16385, 32767, 32768, 32769,
65535, 65536, 65537, 131071, 131072, 131073, dataSize - 1, dataSize}
// Open stream with a seek of underlyingOffset
open := func(underlyingOffset int64) (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):])), nil
}
// Now try decoding it with a open/seek
for _, offset := range trials {
rc, err := c.DecryptDataSeek(open, int64(offset))
assert.NoError(t, err)
seekedDecrypted, err := ioutil.ReadAll(rc)
assert.NoError(t, err)
assert.Equal(t, plaintext[offset:], seekedDecrypted)
}
// Now try decoding it with a single open and lots of seeks
rc, err := c.DecryptDataSeek(open, 0)
for _, offset := range trials {
_, err := rc.Seek(int64(offset), io.SeekStart)
assert.NoError(t, err)
seekedDecrypted, err := ioutil.ReadAll(rc)
assert.NoError(t, err)
assert.Equal(t, plaintext[offset:], seekedDecrypted)
}
}
func TestDecrypterRead(t *testing.T) {
c, err := newCipher(NameEncryptionStandard, "", "")
assert.NoError(t, err)

48
crypt/crypt.go
View File

@ -4,7 +4,6 @@ package crypt
import (
"fmt"
"io"
"io/ioutil"
"path"
"sync"
@ -298,7 +297,7 @@ func (o *Object) Hash(hash fs.HashType) (string, error) {
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
var offset int64
for _, option := range options {
switch x := option.(type) {
@ -310,46 +309,17 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
}
}
}
in, err := o.Object.Open()
rc, err = o.f.cipher.DecryptDataSeek(func(underlyingOffset int64) (io.ReadCloser, error) {
if underlyingOffset == 0 {
// Open with no seek
return o.Object.Open()
}
// Open stream with a seek of underlyingOffset
return o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
}, offset)
if err != nil {
return nil, err
}
// This reads the header and checks it is OK
rc, err := o.f.cipher.DecryptData(in)
if err != nil {
return nil, err
}
// If seeking required, then...
if offset != 0 {
// FIXME could cache the unseeked decrypter as we re-read the header on every seek
decrypter := rc.(*decrypter)
// Seek the decrypter and work out where to seek the
// underlying file and how many bytes to discard
underlyingOffset, discard := decrypter.seek(offset)
// Re-open stream with a seek of underlyingOffset
err = in.Close()
if err != nil {
return nil, err
}
in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
if err != nil {
return nil, err
}
// Update the stream
decrypter.rc = in
// Discard the bytes
_, err = io.CopyN(ioutil.Discard, decrypter, discard)
if err != nil {
return nil, err
}
}
return rc, err
}