From aef2ac5c04cfc21617859aa10addec9b375242f6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 10 Sep 2016 11:29:57 +0100 Subject: [PATCH] Add options for Open and implement Range for all remotes --- amazonclouddrive/amazonclouddrive.go | 7 +- amazonclouddrive/amazonclouddrive_test.go | 1 + b2/b2.go | 3 +- b2/b2_test.go | 1 + crypt/cipher.go | 38 ++++- crypt/cipher_test.go | 138 ++++++++++++++++++ crypt/crypt.go | 54 ++++++- crypt/crypt2_test.go | 1 + crypt/crypt_test.go | 1 + drive/drive.go | 22 +-- drive/drive_test.go | 1 + dropbox/dropbox.go | 17 ++- dropbox/dropbox_test.go | 1 + fs/fs.go | 2 +- fs/lister_test.go | 22 +-- fs/options.go | 137 +++++++++++++++++ fstest/fstests/fstests.go | 65 ++++++--- googlecloudstorage/googlecloudstorage.go | 6 +- googlecloudstorage/googlecloudstorage_test.go | 1 + hubic/hubic_test.go | 1 + local/local.go | 26 +++- local/local_test.go | 1 + onedrive/onedrive.go | 7 +- onedrive/onedrive_test.go | 1 + rest/rest.go | 27 +++- s3/s3.go | 13 +- s3/s3_test.go | 1 + swift/swift.go | 6 +- swift/swift_test.go | 1 + yandex/api/download.go | 6 +- yandex/api/performdownload.go | 12 +- yandex/yandex.go | 4 +- yandex/yandex_test.go | 1 + 33 files changed, 547 insertions(+), 78 deletions(-) create mode 100644 fs/options.go diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index 9c4979b42..2b1e80688 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -786,18 +786,19 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { bigObject := o.Size() >= int64(tempLinkThreshold) if bigObject { fs.Debug(o, "Dowloading large object via tempLink") } file := acd.File{Node: o.info} var resp *http.Response + headers := fs.OpenOptionHeaders(options) err = o.fs.pacer.Call(func() (bool, error) { if !bigObject { - in, resp, err = file.Open() + in, resp, err = file.OpenHeaders(headers) } else { - in, resp, err = file.OpenTempURL(o.fs.noAuthClient) + in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers) } return o.fs.shouldRetry(resp, err) }) diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go index 3748d3f53..44dfca0fc 100644 --- a/amazonclouddrive/amazonclouddrive_test.go +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/b2/b2.go b/b2/b2.go index e0781ecf9..5b9558b20 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -1082,11 +1082,12 @@ func (file *openFile) Close() (err error) { var _ io.ReadCloser = &openFile{} // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { opts := rest.Opts{ Method: "GET", Absolute: true, Path: o.fs.info.DownloadURL, + Options: options, } // Download by id if set otherwise by name if o.id != "" { diff --git a/b2/b2_test.go b/b2/b2_test.go index fb3f1c808..519f99c55 100644 --- a/b2/b2_test.go +++ b/b2/b2_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/crypt/cipher.go b/crypt/cipher.go index 4ac3de52a..102144895 100644 --- a/crypt/cipher.go +++ b/crypt/cipher.go @@ -355,9 +355,9 @@ func (n *nonce) fromBuf(buf []byte) { } } -// increment to add 1 to the nonce -func (n *nonce) increment() { - for i := 0; i < len(*n); i++ { +// carry 1 up the nonce from position i +func (n *nonce) carry(i int) { + for ; i < len(*n); i++ { digit := (*n)[i] newDigit := digit + 1 (*n)[i] = newDigit @@ -368,6 +368,27 @@ func (n *nonce) increment() { } } +// increment to add 1 to the nonce +func (n *nonce) increment() { + n.carry(0) +} + +// add an uint64 to the nonce +func (n *nonce) add(x uint64) { + carry := uint16(0) + for i := 0; i < 8; i++ { + digit := (*n)[i] + xDigit := byte(x) + x >>= 8 + carry += uint16(digit) + uint16(xDigit) + (*n)[i] = byte(carry) + carry >>= 8 + } + if carry != 0 { + n.carry(8) + } +} + // encrypter encrypts an io.Reader on the fly type encrypter struct { in io.Reader @@ -528,6 +549,17 @@ func (fh *decrypter) Read(p []byte) (n int, err error) { return n, nil } +// seek the decryption forwards the amount given +// +// returns an offset for the underlying rc to be seeked and the number +// of bytes to be discarded +func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) { + blocks, discard := offset/blockDataSize, offset%blockDataSize + underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize) + fh.nonce.add(uint64(blocks)) + return +} + // finish sets the final error and tidies up func (fh *decrypter) finish(err error) error { if fh.err != nil { diff --git a/crypt/cipher_test.go b/crypt/cipher_test.go index 6ec09cc10..405c04870 100644 --- a/crypt/cipher_test.go +++ b/crypt/cipher_test.go @@ -464,6 +464,144 @@ func TestNonceIncrement(t *testing.T) { } } +func TestNonceAdd(t *testing.T) { + for _, test := range []struct { + add uint64 + in nonce + out nonce + }{ + { + 0x01, + nonce{0x00}, + nonce{0x01}, + }, + { + 0xFF, + nonce{0xFF}, + nonce{0xFE, 0x01}, + }, + { + 0xFFFF, + nonce{0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0x01}, + }, + { + 0xFFFFFF, + nonce{0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFe, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, + }, + { + 0xFFFFFFFFFFFFFFFF, + nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + }, + } { + x := test.in + x.add(test.add) + assert.Equal(t, test.out, x) + } +} + // randomSource can read or write a random sequence type randomSource struct { counter int64 diff --git a/crypt/crypt.go b/crypt/crypt.go index 69dfbb52a..5c39334bc 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -4,6 +4,7 @@ package crypt import ( "fmt" "io" + "io/ioutil" "path" "sync" @@ -297,12 +298,59 @@ func (o *Object) Hash(hash fs.HashType) (string, error) { } // Open opens the file for read. Call Close() on the returned io.ReadCloser -func (o *Object) Open() (io.ReadCloser, error) { +func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { + var offset int64 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + default: + if option.Mandatory() { + fs.Log(o, "Unsupported mandatory option: %v", option) + } + } + } in, err := o.Object.Open() if err != nil { - return in, err + return nil, err } - return o.f.cipher.DecryptData(in) + + // This reads the header and checks it is OK + rc, err := o.f.cipher.DecryptData(in) + if err != nil { + return nil, err + } + + // If seeking required, then... + if offset != 0 { + // FIXME could cache the unseeked decrypter as we re-read the header on every seek + decrypter := rc.(*decrypter) + + // Seek the decrypter and work out where to seek the + // underlying file and how many bytes to discard + underlyingOffset, discard := decrypter.seek(offset) + + // Re-open stream with a seek of underlyingOffset + err = in.Close() + if err != nil { + return nil, err + } + in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset}) + if err != nil { + return nil, err + } + + // Update the stream + decrypter.rc = in + + // Discard the bytes + _, err = io.CopyN(ioutil.Discard, decrypter, discard) + if err != nil { + return nil, err + } + } + + return rc, err } // Update in to the object with the modTime given of the given size diff --git a/crypt/crypt2_test.go b/crypt/crypt2_test.go index 25ae95323..ffd359b24 100644 --- a/crypt/crypt2_test.go +++ b/crypt/crypt2_test.go @@ -51,6 +51,7 @@ func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek2(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go index e20ad1fa8..d7481f478 100644 --- a/crypt/crypt_test.go +++ b/crypt/crypt_test.go @@ -51,6 +51,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/drive/drive.go b/drive/drive.go index 632b0e070..75c140484 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -827,7 +827,7 @@ func (o *Object) Size() int64 { if o.isDocument && o.bytes < 0 { // If it is a google doc then we must HEAD it to see // how big it is - res, err := o.httpResponse("HEAD") + _, res, err := o.httpResponse("HEAD", nil) if err != nil { fs.ErrorLog(o, "Error reading size: %v", err) return 0 @@ -929,22 +929,23 @@ func (o *Object) Storable() bool { // httpResponse gets an http.Response object for the object o.url // using the method passed in -func (o *Object) httpResponse(method string) (res *http.Response, err error) { +func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) { if o.url == "" { - return nil, errors.New("forbidden to download - check sharing permission") + return nil, nil, errors.New("forbidden to download - check sharing permission") } - req, err := http.NewRequest(method, o.url, nil) + req, err = http.NewRequest(method, o.url, nil) if err != nil { - return nil, err + return req, nil, err } + fs.OpenOptionAddHTTPHeaders(req.Header, options) err = o.fs.pacer.Call(func() (bool, error) { res, err = o.fs.client.Do(req) return shouldRetry(err) }) if err != nil { - return nil, err + return req, nil, err } - return res, nil + return req, res, nil } // openFile represents an Object open for reading @@ -979,12 +980,13 @@ func (file *openFile) Close() (err error) { var _ io.ReadCloser = &openFile{} // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { - res, err := o.httpResponse("GET") +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + req, res, err := o.httpResponse("GET", options) if err != nil { return nil, err } - if res.StatusCode != 200 { + _, isRanging := req.Header["Range"] + if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) { _ = res.Body.Close() // ignore error return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status) } diff --git a/drive/drive_test.go b/drive/drive_test.go index e7f790f86..1a92c3de8 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 4390b77bd..9c0e86f36 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -710,8 +710,21 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { - in, _, err = o.fs.db.Download(o.remotePath(), "", 0) +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + // FIXME should send a patch for dropbox module which allow setting headers + var offset int64 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + default: + if option.Mandatory() { + fs.Log(o, "Unsupported mandatory option: %v", option) + } + } + } + + in, _, err = o.fs.db.Download(o.remotePath(), "", offset) if dropboxErr, ok := err.(*dropbox.Error); ok { // Dropbox return 461 for copyright violation so don't // attempt to retry this error diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 386949d46..e41ffa6f9 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/fs/fs.go b/fs/fs.go index 0f981be0a..049503c08 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -172,7 +172,7 @@ type Object interface { SetModTime(time.Time) error // Open opens the file for read. Call Close() on the returned io.ReadCloser - Open() (io.ReadCloser, error) + Open(options ...OpenOption) (io.ReadCloser, error) // Update in to the object with the modTime given of the given size Update(in io.Reader, src ObjectInfo) error diff --git a/fs/lister_test.go b/fs/lister_test.go index 3414f89ad..aab4a6e07 100644 --- a/fs/lister_test.go +++ b/fs/lister_test.go @@ -21,17 +21,17 @@ var errNotImpl = errors.New("not implemented") type mockObject string -func (o mockObject) String() string { return string(o) } -func (o mockObject) Fs() Info { return nil } -func (o mockObject) Remote() string { return string(o) } -func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl } -func (o mockObject) ModTime() (t time.Time) { return t } -func (o mockObject) Size() int64 { return 0 } -func (o mockObject) Storable() bool { return true } -func (o mockObject) SetModTime(time.Time) error { return errNotImpl } -func (o mockObject) Open() (io.ReadCloser, error) { return nil, errNotImpl } -func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl } -func (o mockObject) Remove() error { return errNotImpl } +func (o mockObject) String() string { return string(o) } +func (o mockObject) Fs() Info { return nil } +func (o mockObject) Remote() string { return string(o) } +func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl } +func (o mockObject) ModTime() (t time.Time) { return t } +func (o mockObject) Size() int64 { return 0 } +func (o mockObject) Storable() bool { return true } +func (o mockObject) SetModTime(time.Time) error { return errNotImpl } +func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl } +func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl } +func (o mockObject) Remove() error { return errNotImpl } type mockFs struct { listFn func(o ListOpts, dir string) diff --git a/fs/options.go b/fs/options.go new file mode 100644 index 000000000..e1f24c004 --- /dev/null +++ b/fs/options.go @@ -0,0 +1,137 @@ +// Define the options for Open + +package fs + +import ( + "fmt" + "net/http" + "strconv" +) + +// OpenOption is an interface describing options for Open +type OpenOption interface { + fmt.Stringer + + // Header returns the option as an HTTP header + Header() (key string, value string) + + // Mandatory returns whether this option can be ignored or not + Mandatory() bool +} + +// RangeOption defines an HTTP Range option with start and end. If +// either start or end are < 0 then they will be omitted. +type RangeOption struct { + Start int64 + End int64 +} + +// Header formats the option as an http header +func (o *RangeOption) Header() (key string, value string) { + key = "Range" + value = "bytes=" + if o.Start >= 0 { + value += strconv.FormatInt(o.Start, 64) + + } + value += "-" + if o.End >= 0 { + value += strconv.FormatInt(o.End, 64) + } + return key, value +} + +// String formats the option into human readable form +func (o *RangeOption) String() string { + return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o *RangeOption) Mandatory() bool { + return false +} + +// SeekOption defines an HTTP Range option with start only. +type SeekOption struct { + Offset int64 +} + +// Header formats the option as an http header +func (o *SeekOption) Header() (key string, value string) { + key = "Range" + value = fmt.Sprintf("bytes=%d-", o.Offset) + return key, value +} + +// String formats the option into human readable form +func (o *SeekOption) String() string { + return fmt.Sprintf("SeekOption(%d)", o.Offset) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o *SeekOption) Mandatory() bool { + return true +} + +// HTTPOption defines a general purpose HTTP option +type HTTPOption struct { + Key string + Value string +} + +// Header formats the option as an http header +func (o *HTTPOption) Header() (key string, value string) { + return o.Key, o.Value +} + +// String formats the option into human readable form +func (o *HTTPOption) String() string { + return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o *HTTPOption) Mandatory() bool { + return false +} + +// OpenOptionAddHeaders adds each header found in options to the +// headers map provided the key was non empty. +func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) { + for _, option := range options { + key, value := option.Header() + if key != "" && value != "" { + headers[key] = value + } + } +} + +// OpenOptionHeaders adds each header found in options to the +// headers map provided the key was non empty. +// +// It returns a nil map if options was empty +func OpenOptionHeaders(options []OpenOption) (headers map[string]string) { + if len(options) == 0 { + return nil + } + headers = make(map[string]string, len(options)) + OpenOptionAddHeaders(options, headers) + return headers +} + +// OpenOptionAddHTTPHeaders Sets each header found in options to the +// http.Header map provided the key was non empty. +func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) { + for _, option := range options { + key, value := option.Header() + if key != "" && value != "" { + headers.Set(key, value) + } + } +} + +// check interface +var ( + _ OpenOption = (*RangeOption)(nil) + _ OpenOption = (*SeekOption)(nil) + _ OpenOption = (*HTTPOption)(nil) +) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 0d6df66e6..2d7626e28 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "os" "path" "sort" @@ -37,14 +38,16 @@ var ( ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), Path: "file name.txt", } - file2 = fstest.Item{ + file1Contents = "" + file2 = fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`, } - verbose = flag.Bool("verbose", false, "Set to enable logging") - dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info") - dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info") + file2Contents = "" + verbose = flag.Bool("verbose", false, "Set to enable logging") + dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info") + dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info") ) // ExtraConfigItem describes a config item added on the fly while testing @@ -195,9 +198,10 @@ func findObject(t *testing.T, Name string) fs.Object { return obj } -func testPut(t *testing.T, file *fstest.Item) { +func testPut(t *testing.T, file *fstest.Item) string { again: - buf := bytes.NewBufferString(fstest.RandomString(100)) + contents := fstest.RandomString(100) + buf := bytes.NewBufferString(contents) hash := fs.NewMultiHasher() in := io.TeeReader(buf, hash) @@ -222,24 +226,25 @@ again: // Re-read the object and check again obj = findObject(t, file.Path) file.Check(t, obj, remote.Precision()) + return contents } // TestFsPutFile1 tests putting a file func TestFsPutFile1(t *testing.T) { skipIfNotOk(t) - testPut(t, &file1) + file1Contents = testPut(t, &file1) } // TestFsPutFile2 tests putting a file into a subdirectory func TestFsPutFile2(t *testing.T) { skipIfNotOk(t) - testPut(t, &file2) + file2Contents = testPut(t, &file2) } // TestFsUpdateFile1 tests updating file1 with new contents func TestFsUpdateFile1(t *testing.T) { skipIfNotOk(t) - testPut(t, &file1) + file1Contents = testPut(t, &file1) // Note that the next test will check there are no duplicated file names } @@ -541,42 +546,56 @@ func TestObjectSize(t *testing.T) { assert.Equal(t, file1.Size, obj.Size()) } +// read the contents of an object as a string +func readObject(t *testing.T, obj fs.Object, options ...fs.OpenOption) string { + in, err := obj.Open(options...) + require.NoError(t, err) + contents, err := ioutil.ReadAll(in) + require.NoError(t, err) + err = in.Close() + require.NoError(t, err) + return string(contents) +} + // TestObjectOpen tests that Open works func TestObjectOpen(t *testing.T) { skipIfNotOk(t) obj := findObject(t, file1.Path) - in, err := obj.Open() - require.NoError(t, err) - hasher := fs.NewMultiHasher() - n, err := io.Copy(hasher, in) - require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err)) - require.Equal(t, file1.Size, n, "Read wrong number of bytes") - err = in.Close() - require.NoError(t, err) - // Check content of file by comparing the calculated hashes - for hashType, got := range hasher.Sums() { - assert.Equal(t, file1.Hashes[hashType], got) - } + assert.Equal(t, file1Contents, readObject(t, obj), "contents of file1 differ") +} +// TestObjectOpenSeek tests that Open works with Seek +func TestObjectOpenSeek(t *testing.T) { + skipIfNotOk(t) + obj := findObject(t, file1.Path) + assert.Equal(t, file1Contents[50:], readObject(t, obj, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") } // TestObjectUpdate tests that Update works func TestObjectUpdate(t *testing.T) { skipIfNotOk(t) - buf := bytes.NewBufferString(fstest.RandomString(200)) + contents := fstest.RandomString(200) + buf := bytes.NewBufferString(contents) hash := fs.NewMultiHasher() in := io.TeeReader(buf, hash) file1.Size = int64(buf.Len()) obj := findObject(t, file1.Path) - obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, file1.Size, true, nil, obj.Fs()) + obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs()) err := obj.Update(in, obji) require.NoError(t, err) file1.Hashes = hash.Sums() + + // check the object has been updated file1.Check(t, obj, remote.Precision()) + // Re-read the object and check again obj = findObject(t, file1.Path) file1.Check(t, obj, remote.Precision()) + + // check contents correct + assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ") + file1Contents = contents } // TestObjectStorable tests that Storable works diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index f46fb6702..fb87faca6 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -651,16 +651,18 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { req, err := http.NewRequest("GET", o.url, nil) if err != nil { return nil, err } + fs.OpenOptionAddHTTPHeaders(req.Header, options) res, err := o.fs.client.Do(req) if err != nil { return nil, err } - if res.StatusCode != 200 { + _, isRanging := req.Header["Range"] + if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) { _ = res.Body.Close() // ignore error return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status) } diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index acbe1508a..24147e3a1 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go index 7159b0b82..124bccb67 100644 --- a/hubic/hubic_test.go +++ b/hubic/hubic_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/local/local.go b/local/local.go index 231f4965e..bf058ab4b 100644 --- a/local/local.go +++ b/local/local.go @@ -585,18 +585,36 @@ func (file *localOpenFile) Close() (err error) { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { - in, err = os.Open(o.path) +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + var offset int64 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + default: + if option.Mandatory() { + fs.Log(o, "Unsupported mandatory option: %v", option) + } + } + } + + fd, err := os.Open(o.path) if err != nil { return } + if offset != 0 { + // seek the object + _, err = fd.Seek(offset, 0) + // don't attempt to make checksums + return fd, err + } // Update the md5sum as we go along in = &localOpenFile{ o: o, - in: in, + in: fd, hash: fs.NewMultiHasher(), } - return + return in, nil } // mkdirAll makes all the directories needed to store the object diff --git a/local/local_test.go b/local/local_test.go index a4c9c4b00..eb9f91a57 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index e870c6377..35d2d2f34 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -775,14 +775,15 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { if o.id == "" { return nil, errors.New("can't download - no id") } var resp *http.Response opts := rest.Opts{ - Method: "GET", - Path: "/drive/items/" + o.id + "/content", + Method: "GET", + Path: "/drive/items/" + o.id + "/content", + Options: options, } err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.Call(&opts) diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go index 0c3807c91..2fdcab36d 100644 --- a/onedrive/onedrive_test.go +++ b/onedrive/onedrive_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/rest/rest.go b/rest/rest.go index 4963347de..5907074c0 100644 --- a/rest/rest.go +++ b/rest/rest.go @@ -83,6 +83,7 @@ type Opts struct { ExtraHeaders map[string]string UserName string // username for Basic Auth Password string // password for Basic Auth + Options []fs.OpenOption } // DecodeJSON decodes resp.Body into result @@ -92,6 +93,27 @@ func DecodeJSON(resp *http.Response, result interface{}) (err error) { return decoder.Decode(result) } +// Make a new http client which resets the headers passed in on redirect +func clientWithHeaderReset(c *http.Client, headers map[string]string) *http.Client { + if len(headers) == 0 { + return c + } + clientCopy := *c + clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + // Reset the headers in the new request + for k, v := range headers { + if v != "" { + req.Header.Add(k, v) + } + } + return nil + } + return &clientCopy +} + // Call makes the call and returns the http.Response // // if err != nil then resp.Body will need to be closed @@ -136,6 +158,8 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { headers[k] = v } } + // add any options to the headers + fs.OpenOptionAddHeaders(opts.Options, headers) // Now set the headers for k, v := range headers { if v != "" { @@ -145,8 +169,9 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { if opts.UserName != "" || opts.Password != "" { req.SetBasicAuth(opts.UserName, opts.Password) } + c := clientWithHeaderReset(api.c, headers) api.mu.RUnlock() - resp, err = api.c.Do(req) + resp, err = c.Do(req) api.mu.RLock() if err != nil { return nil, err diff --git a/s3/s3.go b/s3/s3.go index c8d4ef597..9c56596ff 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -845,12 +845,23 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { key := o.fs.root + o.remote req := s3.GetObjectInput{ Bucket: &o.fs.bucket, Key: &key, } + for _, option := range options { + switch option.(type) { + case *fs.RangeOption, *fs.SeekOption: + _, value := option.Header() + req.Range = &value + default: + if option.Mandatory() { + fs.Log(o, "Unsupported mandatory option: %v", option) + } + } + } resp, err := o.fs.c.GetObject(&req) if err != nil { return nil, err diff --git a/s3/s3_test.go b/s3/s3_test.go index 6c6ec7426..fdb646fd8 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/swift/swift.go b/swift/swift.go index 32dcc0599..429a8c241 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -629,8 +629,10 @@ func (o *Object) Storable() bool { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { - in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, true, nil) +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + headers := fs.OpenOptionHeaders(options) + _, isRanging := headers["Range"] + in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers) return } diff --git a/swift/swift_test.go b/swift/swift_test.go index 1bc0c56e0..27d78b45c 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } diff --git a/yandex/api/download.go b/yandex/api/download.go index 10c01d952..310d34679 100644 --- a/yandex/api/download.go +++ b/yandex/api/download.go @@ -13,13 +13,13 @@ type DownloadResponse struct { Templated bool `json:"templated"` } -// Download will get specified data from Yandex.Disk. -func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer +// Download will get specified data from Yandex.Disk supplying the extra headers +func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer ur, err := c.DownloadRequest(remotePath) if err != nil { return nil, err } - return c.PerformDownload(ur.HRef) + return c.PerformDownload(ur.HRef, headers) } // DownloadRequest will make an download request and return a URL to download data to. diff --git a/yandex/api/performdownload.go b/yandex/api/performdownload.go index d9c74f849..20e802637 100644 --- a/yandex/api/performdownload.go +++ b/yandex/api/performdownload.go @@ -8,13 +8,18 @@ import ( "github.com/pkg/errors" ) -// PerformDownload does the actual download via unscoped PUT request. -func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) { +// PerformDownload does the actual download via unscoped GET request. +func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } + // Set any extra headers + for k, v := range headers { + req.Header.Set(k, v) + } + //c.setRequestScope(req) resp, err := c.HTTPClient.Do(req) @@ -22,7 +27,8 @@ func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) { return nil, err } - if resp.StatusCode != 200 { + _, isRanging := req.Header["Range"] + if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) { defer CheckClose(resp.Body, &err) body, err := ioutil.ReadAll(resp.Body) if err != nil { diff --git a/yandex/yandex.go b/yandex/yandex.go index 1e7c10509..67247fe8c 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -487,8 +487,8 @@ func (o *Object) ModTime() time.Time { } // Open an object for read -func (o *Object) Open() (in io.ReadCloser, err error) { - return o.fs.yd.Download(o.remotePath()) +func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { + return o.fs.yd.Download(o.remotePath(), fs.OpenOptionHeaders(options)) } // Remove an object diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go index 235170d3a..cd35dc043 100644 --- a/yandex/yandex_test.go +++ b/yandex/yandex_test.go @@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }