github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-17 17:30:37 +01:00
Code Releases Activity

drive: add --drive-chunk-size and --drive-upload-cutoff parameters

Browse Source
This commit is contained in:
Nick Craig-Wood 2015-03-14 17:55:38 +00:00
parent 5cc5429f99
commit 7c120b8bc5
2 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
drive/drive.go
View File

@ -39,6 +39,10 @@ const (
var (
// Flags
driveFullList = pflag.BoolP("drive-full-list", "", true, "Use a full listing for directory list. More data but usually quicker.")
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize = fs.SizeSuffix(256 * 1024)
driveUploadCutoff = chunkSize
// Description of how to auth for this app
driveAuth = &googleauth.Auth{
Scope: "https://www.googleapis.com/auth/drive",
@ -63,6 +67,8 @@ func init() {
Help: "Google Application Client Secret - leave blank to use rclone's.",
}},
})
pflag.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
}
// FsDrive represents a remote drive server
@ -293,8 +299,27 @@ OUTER:
return
}
// Returns true of x is a power of 2 or zero
func isPowerOfTwo(x int64) bool {
switch {
case x == 0:
return true
case x < 0:
return false
default:
return (x & (x - 1)) == 0
}
}
// NewFs contstructs an FsDrive from the path, container:path
func NewFs(name, path string) (fs.Fs, error) {
if !isPowerOfTwo(int64(chunkSize)) {
return nil, fmt.Errorf("drive: chunk size %v isn't a power of two", chunkSize)
}
if chunkSize < 256*1024 {
return nil, fmt.Errorf("drive: chunk size can't be less than 256k - was %v", chunkSize)
}
t, err := driveAuth.NewTransport(name)
if err != nil {
return nil, err
@ -730,7 +755,7 @@ func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64
}
var info *drive.File
if false {
if size == 0 || size < int64(driveUploadCutoff) {
// Make the API request to upload metadata and file data.
// Don't retry, return a retry error instead
f.beginCall()
@ -970,7 +995,7 @@ func (o *FsObjectDrive) Update(in io.Reader, modTime time.Time, size int64) erro
// Make the API request to upload metadata and file data.
var err error
var info *drive.File
if false {
if size == 0 || size < int64(driveUploadCutoff) {
// Don't retry, return a retry error instead
o.drive.beginCall()
info, err = o.drive.svc.Files.Update(updateInfo.Id, updateInfo).SetModifiedDate(true).Media(in).Do()

8
drive/upload.go
View File

@ -31,10 +31,6 @@ const (
// Number of times to try each chunk
maxTries = 10
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize int64 = 1 << 18
)
// resumableUpload is used by the generated APIs to provide resumable uploads.
@ -194,8 +190,8 @@ func (rx *resumableUpload) Upload() (*drive.File, error) {
var StatusCode int
for start < rx.ContentLength {
reqSize := rx.ContentLength - start
if reqSize >= chunkSize {
reqSize = chunkSize
if reqSize >= int64(chunkSize) {
reqSize = int64(chunkSize)
} else {
buf = buf[:reqSize]
}