github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-22 13:03:02 +01:00
Code Releases Activity

Add options for Open and implement Range for all remotes

Browse Source
This commit is contained in:
Nick Craig-Wood 2016-09-10 11:29:57 +01:00
parent 75e5e59385
commit aef2ac5c04
33 changed files with 547 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
amazonclouddrive/amazonclouddrive.go
View File

@ -786,18 +786,19 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
bigObject := o.Size() >= int64(tempLinkThreshold)
if bigObject {
fs.Debug(o, "Dowloading large object via tempLink")
}
file := acd.File{Node: o.info}
var resp *http.Response
headers := fs.OpenOptionHeaders(options)
err = o.fs.pacer.Call(func() (bool, error) {
if !bigObject {
in, resp, err = file.Open()
in, resp, err = file.OpenHeaders(headers)
} else {
in, resp, err = file.OpenTempURL(o.fs.noAuthClient)
in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
}
return o.fs.shouldRetry(resp, err)
})

1
amazonclouddrive/amazonclouddrive_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

3
b2/b2.go
View File

@ -1082,11 +1082,12 @@ func (file *openFile) Close() (err error) {
var _ io.ReadCloser = &openFile{}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
opts := rest.Opts{
Method: "GET",
Absolute: true,
Path: o.fs.info.DownloadURL,
Options: options,
}
// Download by id if set otherwise by name
if o.id != "" {

1
b2/b2_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

38
crypt/cipher.go
View File

@ -355,9 +355,9 @@ func (n *nonce) fromBuf(buf []byte) {
}
}
// increment to add 1 to the nonce
func (n *nonce) increment() {
for i := 0; i < len(*n); i++ {
// carry 1 up the nonce from position i
func (n *nonce) carry(i int) {
for ; i < len(*n); i++ {
digit := (*n)[i]
newDigit := digit + 1
(*n)[i] = newDigit
@ -368,6 +368,27 @@ func (n *nonce) increment() {
}
}
// increment to add 1 to the nonce
func (n *nonce) increment() {
n.carry(0)
}
// add an uint64 to the nonce
func (n *nonce) add(x uint64) {
carry := uint16(0)
for i := 0; i < 8; i++ {
digit := (*n)[i]
xDigit := byte(x)
x >>= 8
carry += uint16(digit) + uint16(xDigit)
(*n)[i] = byte(carry)
carry >>= 8
}
if carry != 0 {
n.carry(8)
}
}
// encrypter encrypts an io.Reader on the fly
type encrypter struct {
in io.Reader
@ -528,6 +549,17 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
return n, nil
}
// seek the decryption forwards the amount given
//
// returns an offset for the underlying rc to be seeked and the number
// of bytes to be discarded
func (fh *decrypter) seek(offset int64) (underlyingOffset int64, discard int64) {
blocks, discard := offset/blockDataSize, offset%blockDataSize
underlyingOffset = int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
fh.nonce.add(uint64(blocks))
return
}
// finish sets the final error and tidies up
func (fh *decrypter) finish(err error) error {
if fh.err != nil {

138
crypt/cipher_test.go
View File

@ -464,6 +464,144 @@ func TestNonceIncrement(t *testing.T) {
}
}
func TestNonceAdd(t *testing.T) {
for _, test := range []struct {
add uint64
in nonce
out nonce
}{
{
0x01,
nonce{0x00},
nonce{0x01},
},
{
0xFF,
nonce{0xFF},
nonce{0xFE, 0x01},
},
{
0xFFFF,
nonce{0xFF, 0xFF},
nonce{0xFE, 0xFF, 0x01},
},
{
0xFFFFFF,
nonce{0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFe, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
},
{
0xFFFFFFFFFFFFFFFF,
nonce{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
nonce{0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
},
} {
x := test.in
x.add(test.add)
assert.Equal(t, test.out, x)
}
}
// randomSource can read or write a random sequence
type randomSource struct {
counter int64

54
crypt/crypt.go
View File

@ -4,6 +4,7 @@ package crypt
import (
"fmt"
"io"
"io/ioutil"
"path"
"sync"
@ -297,12 +298,59 @@ func (o *Object) Hash(hash fs.HashType) (string, error) {
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open() (io.ReadCloser, error) {
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
in, err := o.Object.Open()
if err != nil {
return in, err
return nil, err
}
return o.f.cipher.DecryptData(in)
// This reads the header and checks it is OK
rc, err := o.f.cipher.DecryptData(in)
if err != nil {
return nil, err
}
// If seeking required, then...
if offset != 0 {
// FIXME could cache the unseeked decrypter as we re-read the header on every seek
decrypter := rc.(*decrypter)
// Seek the decrypter and work out where to seek the
// underlying file and how many bytes to discard
underlyingOffset, discard := decrypter.seek(offset)
// Re-open stream with a seek of underlyingOffset
err = in.Close()
if err != nil {
return nil, err
}
in, err := o.Object.Open(&fs.SeekOption{Offset: underlyingOffset})
if err != nil {
return nil, err
}
// Update the stream
decrypter.rc = in
// Discard the bytes
_, err = io.CopyN(ioutil.Discard, decrypter, discard)
if err != nil {
return nil, err
}
}
return rc, err
}
// Update in to the object with the modTime given of the given size

1
crypt/crypt2_test.go
View File

@ -51,6 +51,7 @@ func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek2(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate2(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable2(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile2(t *testing.T) { fstests.TestFsIsFile(t) }

1
crypt/crypt_test.go
View File

@ -51,6 +51,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

22
drive/drive.go
View File

@ -827,7 +827,7 @@ func (o *Object) Size() int64 {
if o.isDocument && o.bytes < 0 {
// If it is a google doc then we must HEAD it to see
// how big it is
res, err := o.httpResponse("HEAD")
_, res, err := o.httpResponse("HEAD", nil)
if err != nil {
fs.ErrorLog(o, "Error reading size: %v", err)
return 0
@ -929,22 +929,23 @@ func (o *Object) Storable() bool {
// httpResponse gets an http.Response object for the object o.url
// using the method passed in
func (o *Object) httpResponse(method string) (res *http.Response, err error) {
func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
if o.url == "" {
return nil, errors.New("forbidden to download - check sharing permission")
return nil, nil, errors.New("forbidden to download - check sharing permission")
}
req, err := http.NewRequest(method, o.url, nil)
req, err = http.NewRequest(method, o.url, nil)
if err != nil {
return nil, err
return req, nil, err
}
fs.OpenOptionAddHTTPHeaders(req.Header, options)
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req)
return shouldRetry(err)
})
if err != nil {
return nil, err
return req, nil, err
}
return res, nil
return req, res, nil
}
// openFile represents an Object open for reading
@ -979,12 +980,13 @@ func (file *openFile) Close() (err error) {
var _ io.ReadCloser = &openFile{}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
res, err := o.httpResponse("GET")
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
req, res, err := o.httpResponse("GET", options)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
_, isRanging := req.Header["Range"]
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
_ = res.Body.Close() // ignore error
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
}

1
drive/drive_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

17
dropbox/dropbox.go
View File

@ -710,8 +710,21 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
in, _, err = o.fs.db.Download(o.remotePath(), "", 0)
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
// FIXME should send a patch for dropbox module which allow setting headers
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
in, _, err = o.fs.db.Download(o.remotePath(), "", offset)
if dropboxErr, ok := err.(*dropbox.Error); ok {
// Dropbox return 461 for copyright violation so don't
// attempt to retry this error

1
dropbox/dropbox_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

2
fs/fs.go
View File

@ -172,7 +172,7 @@ type Object interface {
SetModTime(time.Time) error
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open() (io.ReadCloser, error)
Open(options ...OpenOption) (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size
Update(in io.Reader, src ObjectInfo) error

22
fs/lister_test.go
View File

@ -21,17 +21,17 @@ var errNotImpl = errors.New("not implemented")
type mockObject string
func (o mockObject) String() string { return string(o) }
func (o mockObject) Fs() Info { return nil }
func (o mockObject) Remote() string { return string(o) }
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open() (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
func (o mockObject) Remove() error { return errNotImpl }
func (o mockObject) String() string { return string(o) }
func (o mockObject) Fs() Info { return nil }
func (o mockObject) Remote() string { return string(o) }
func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl }
func (o mockObject) ModTime() (t time.Time) { return t }
func (o mockObject) Size() int64 { return 0 }
func (o mockObject) Storable() bool { return true }
func (o mockObject) SetModTime(time.Time) error { return errNotImpl }
func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl }
func (o mockObject) Update(in io.Reader, src ObjectInfo) error { return errNotImpl }
func (o mockObject) Remove() error { return errNotImpl }
type mockFs struct {
listFn func(o ListOpts, dir string)

137
fs/options.go Normal file
View File

@ -0,0 +1,137 @@
// Define the options for Open
package fs
import (
"fmt"
"net/http"
"strconv"
)
// OpenOption is an interface describing options for Open
type OpenOption interface {
fmt.Stringer
// Header returns the option as an HTTP header
Header() (key string, value string)
// Mandatory returns whether this option can be ignored or not
Mandatory() bool
}
// RangeOption defines an HTTP Range option with start and end. If
// either start or end are < 0 then they will be omitted.
type RangeOption struct {
Start int64
End int64
}
// Header formats the option as an http header
func (o *RangeOption) Header() (key string, value string) {
key = "Range"
value = "bytes="
if o.Start >= 0 {
value += strconv.FormatInt(o.Start, 64)
}
value += "-"
if o.End >= 0 {
value += strconv.FormatInt(o.End, 64)
}
return key, value
}
// String formats the option into human readable form
func (o *RangeOption) String() string {
return fmt.Sprintf("RangeOption(%d,%d)", o.Start, o.End)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *RangeOption) Mandatory() bool {
return false
}
// SeekOption defines an HTTP Range option with start only.
type SeekOption struct {
Offset int64
}
// Header formats the option as an http header
func (o *SeekOption) Header() (key string, value string) {
key = "Range"
value = fmt.Sprintf("bytes=%d-", o.Offset)
return key, value
}
// String formats the option into human readable form
func (o *SeekOption) String() string {
return fmt.Sprintf("SeekOption(%d)", o.Offset)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *SeekOption) Mandatory() bool {
return true
}
// HTTPOption defines a general purpose HTTP option
type HTTPOption struct {
Key string
Value string
}
// Header formats the option as an http header
func (o *HTTPOption) Header() (key string, value string) {
return o.Key, o.Value
}
// String formats the option into human readable form
func (o *HTTPOption) String() string {
return fmt.Sprintf("HTTPOption(%q,%q)", o.Key, o.Value)
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o *HTTPOption) Mandatory() bool {
return false
}
// OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
for _, option := range options {
key, value := option.Header()
if key != "" && value != "" {
headers[key] = value
}
}
}
// OpenOptionHeaders adds each header found in options to the
// headers map provided the key was non empty.
//
// It returns a nil map if options was empty
func OpenOptionHeaders(options []OpenOption) (headers map[string]string) {
if len(options) == 0 {
return nil
}
headers = make(map[string]string, len(options))
OpenOptionAddHeaders(options, headers)
return headers
}
// OpenOptionAddHTTPHeaders Sets each header found in options to the
// http.Header map provided the key was non empty.
func OpenOptionAddHTTPHeaders(headers http.Header, options []OpenOption) {
for _, option := range options {
key, value := option.Header()
if key != "" && value != "" {
headers.Set(key, value)
}
}
}
// check interface
var (
_ OpenOption = (*RangeOption)(nil)
_ OpenOption = (*SeekOption)(nil)
_ OpenOption = (*HTTPOption)(nil)
)

65
fstest/fstests/fstests.go
View File

@ -10,6 +10,7 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"sort"
@ -37,14 +38,16 @@ var (
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: "file name.txt",
}
file2 = fstest.Item{
file1Contents = ""
file2 = fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`,
}
verbose = flag.Bool("verbose", false, "Set to enable logging")
dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info")
file2Contents = ""
verbose = flag.Bool("verbose", false, "Set to enable logging")
dumpHeaders = flag.Bool("dump-headers", false, "Dump HTTP headers - may contain sensitive info")
dumpBodies = flag.Bool("dump-bodies", false, "Dump HTTP headers and bodies - may contain sensitive info")
)
// ExtraConfigItem describes a config item added on the fly while testing
@ -195,9 +198,10 @@ func findObject(t *testing.T, Name string) fs.Object {
return obj
}
func testPut(t *testing.T, file *fstest.Item) {
func testPut(t *testing.T, file *fstest.Item) string {
again:
buf := bytes.NewBufferString(fstest.RandomString(100))
contents := fstest.RandomString(100)
buf := bytes.NewBufferString(contents)
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
@ -222,24 +226,25 @@ again:
// Re-read the object and check again
obj = findObject(t, file.Path)
file.Check(t, obj, remote.Precision())
return contents
}
// TestFsPutFile1 tests putting a file
func TestFsPutFile1(t *testing.T) {
skipIfNotOk(t)
testPut(t, &file1)
file1Contents = testPut(t, &file1)
}
// TestFsPutFile2 tests putting a file into a subdirectory
func TestFsPutFile2(t *testing.T) {
skipIfNotOk(t)
testPut(t, &file2)
file2Contents = testPut(t, &file2)
}
// TestFsUpdateFile1 tests updating file1 with new contents
func TestFsUpdateFile1(t *testing.T) {
skipIfNotOk(t)
testPut(t, &file1)
file1Contents = testPut(t, &file1)
// Note that the next test will check there are no duplicated file names
}
@ -541,42 +546,56 @@ func TestObjectSize(t *testing.T) {
assert.Equal(t, file1.Size, obj.Size())
}
// read the contents of an object as a string
func readObject(t *testing.T, obj fs.Object, options ...fs.OpenOption) string {
in, err := obj.Open(options...)
require.NoError(t, err)
contents, err := ioutil.ReadAll(in)
require.NoError(t, err)
err = in.Close()
require.NoError(t, err)
return string(contents)
}
// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
in, err := obj.Open()
require.NoError(t, err)
hasher := fs.NewMultiHasher()
n, err := io.Copy(hasher, in)
require.NoError(t, err, fmt.Sprintf("hasher copy error: %v", err))
require.Equal(t, file1.Size, n, "Read wrong number of bytes")
err = in.Close()
require.NoError(t, err)
// Check content of file by comparing the calculated hashes
for hashType, got := range hasher.Sums() {
assert.Equal(t, file1.Hashes[hashType], got)
}
assert.Equal(t, file1Contents, readObject(t, obj), "contents of file1 differ")
}
// TestObjectOpenSeek tests that Open works with Seek
func TestObjectOpenSeek(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
assert.Equal(t, file1Contents[50:], readObject(t, obj, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
}
// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t)
buf := bytes.NewBufferString(fstest.RandomString(200))
contents := fstest.RandomString(200)
buf := bytes.NewBufferString(contents)
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path)
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, file1.Size, true, nil, obj.Fs())
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
err := obj.Update(in, obji)
require.NoError(t, err)
file1.Hashes = hash.Sums()
// check the object has been updated
file1.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision())
// check contents correct
assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ")
file1Contents = contents
}
// TestObjectStorable tests that Storable works

6
googlecloudstorage/googlecloudstorage.go
View File

@ -651,16 +651,18 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
req, err := http.NewRequest("GET", o.url, nil)
if err != nil {
return nil, err
}
fs.OpenOptionAddHTTPHeaders(req.Header, options)
res, err := o.fs.client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
_, isRanging := req.Header["Range"]
if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
_ = res.Body.Close() // ignore error
return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
}

1
googlecloudstorage/googlecloudstorage_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

1
hubic/hubic_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

26
local/local.go
View File

@ -585,18 +585,36 @@ func (file *localOpenFile) Close() (err error) {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
in, err = os.Open(o.path)
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
fd, err := os.Open(o.path)
if err != nil {
return
}
if offset != 0 {
// seek the object
_, err = fd.Seek(offset, 0)
// don't attempt to make checksums
return fd, err
}
// Update the md5sum as we go along
in = &localOpenFile{
o: o,
in: in,
in: fd,
hash: fs.NewMultiHasher(),
}
return
return in, nil
}
// mkdirAll makes all the directories needed to store the object

1
local/local_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

7
onedrive/onedrive.go
View File

@ -775,14 +775,15 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
if o.id == "" {
return nil, errors.New("can't download - no id")
}
var resp *http.Response
opts := rest.Opts{
Method: "GET",
Path: "/drive/items/" + o.id + "/content",
Method: "GET",
Path: "/drive/items/" + o.id + "/content",
Options: options,
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts)

1
onedrive/onedrive_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

27
rest/rest.go
View File

@ -83,6 +83,7 @@ type Opts struct {
ExtraHeaders map[string]string
UserName string // username for Basic Auth
Password string // password for Basic Auth
Options []fs.OpenOption
}
// DecodeJSON decodes resp.Body into result
@ -92,6 +93,27 @@ func DecodeJSON(resp *http.Response, result interface{}) (err error) {
return decoder.Decode(result)
}
// Make a new http client which resets the headers passed in on redirect
func clientWithHeaderReset(c *http.Client, headers map[string]string) *http.Client {
if len(headers) == 0 {
return c
}
clientCopy := *c
clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
// Reset the headers in the new request
for k, v := range headers {
if v != "" {
req.Header.Add(k, v)
}
}
return nil
}
return &clientCopy
}
// Call makes the call and returns the http.Response
//
// if err != nil then resp.Body will need to be closed
@ -136,6 +158,8 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
headers[k] = v
}
}
// add any options to the headers
fs.OpenOptionAddHeaders(opts.Options, headers)
// Now set the headers
for k, v := range headers {
if v != "" {
@ -145,8 +169,9 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
if opts.UserName != "" || opts.Password != "" {
req.SetBasicAuth(opts.UserName, opts.Password)
}
c := clientWithHeaderReset(api.c, headers)
api.mu.RUnlock()
resp, err = api.c.Do(req)
resp, err = c.Do(req)
api.mu.RLock()
if err != nil {
return nil, err

13
s3/s3.go
View File

@ -845,12 +845,23 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
key := o.fs.root + o.remote
req := s3.GetObjectInput{
Bucket: &o.fs.bucket,
Key: &key,
}
for _, option := range options {
switch option.(type) {
case *fs.RangeOption, *fs.SeekOption:
_, value := option.Header()
req.Range = &value
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
resp, err := o.fs.c.GetObject(&req)
if err != nil {
return nil, err

1
s3/s3_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

6
swift/swift.go
View File

@ -629,8 +629,10 @@ func (o *Object) Storable() bool {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, true, nil)
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
headers := fs.OpenOptionHeaders(options)
_, isRanging := headers["Range"]
in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
return
}

1
swift/swift_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }

6
yandex/api/download.go
View File

@ -13,13 +13,13 @@ type DownloadResponse struct {
Templated bool `json:"templated"`
}
// Download will get specified data from Yandex.Disk.
func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer
// Download will get specified data from Yandex.Disk supplying the extra headers
func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer
ur, err := c.DownloadRequest(remotePath)
if err != nil {
return nil, err
}
return c.PerformDownload(ur.HRef)
return c.PerformDownload(ur.HRef, headers)
}
// DownloadRequest will make an download request and return a URL to download data to.

12
yandex/api/performdownload.go
View File

@ -8,13 +8,18 @@ import (
"github.com/pkg/errors"
)
// PerformDownload does the actual download via unscoped PUT request.
func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) {
// PerformDownload does the actual download via unscoped GET request.
func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
// Set any extra headers
for k, v := range headers {
req.Header.Set(k, v)
}
//c.setRequestScope(req)
resp, err := c.HTTPClient.Do(req)
@ -22,7 +27,8 @@ func (c *Client) PerformDownload(url string) (out io.ReadCloser, err error) {
return nil, err
}
if resp.StatusCode != 200 {
_, isRanging := req.Header["Range"]
if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) {
defer CheckClose(resp.Body, &err)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {

4
yandex/yandex.go
View File

@ -487,8 +487,8 @@ func (o *Object) ModTime() time.Time {
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
return o.fs.yd.Download(o.remotePath())
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
return o.fs.yd.Download(o.remotePath(), fs.OpenOptionHeaders(options))
}
// Remove an object

1
yandex/yandex_test.go
View File

@ -50,6 +50,7 @@ func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }