1
mirror of https://github.com/rclone/rclone synced 2024-10-31 20:16:42 +01:00

Add --buffer-size parameter to control buffer size for copy

This commit is contained in:
Nick Craig-Wood 2017-02-14 19:31:33 +00:00
parent 541929258b
commit 493da54113
3 changed files with 19 additions and 4 deletions

View File

@ -261,6 +261,13 @@ you have a 10 Mbit/s connection and you wish rclone to use half of it
- 5 Mbit/s. This is 5/8 = 0.625MByte/s so you would use a `--bwlimit
0.625M` parameter for rclone.
### --buffer-size=SIZE ###
Use this sized buffer to speed up file transfers. Each `--transfer`
will use this much memory for buffering.
Set to 0 to disable the buffering for the minimum memory use.
### --checkers=N ###
The number of checkers to run in parallel. Checkers do the equality

View File

@ -348,11 +348,15 @@ func NewAccount(in io.ReadCloser, obj Object) *Account {
//
// If the file is above a certain size it adds an Async reader
func NewAccountSizeNameWithBuffer(in io.ReadCloser, size int64, name string) *Account {
const bufSize = 128 * 1024
var buffers int
if size >= int64(Config.BufferSize) {
buffers = int(int64(Config.BufferSize) / bufSize)
} else {
buffers = int(size / bufSize)
}
// On big files add a buffer
if size > 10<<20 {
const memUsed = 16 * 1024 * 1024
const bufSize = 128 * 1024
const buffers = memUsed / bufSize
if buffers > 0 {
newIn, err := newAsyncReader(in, buffers, bufSize)
if err != nil {
Errorf(name, "Failed to make buffer: %v", err)

View File

@ -91,6 +91,7 @@ var (
backupDir = StringP("backup-dir", "", "", "Make backups into hierarchy based in DIR.")
suffix = StringP("suffix", "", "", "Suffix for use with --backup-dir.")
bwLimit BwTimetable
bufferSize SizeSuffix = 16 << 20
// Key to use for password en/decryption.
// When nil, no encryption will be used for saving.
@ -99,6 +100,7 @@ var (
func init() {
VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
VarP(&bufferSize, "buffer-size", "", "Buffer size when copying files.")
}
// crypt internals
@ -213,6 +215,7 @@ type ConfigInfo struct {
DataRateUnit string
BackupDir string
Suffix string
BufferSize SizeSuffix
}
// Return the path to the configuration file
@ -358,6 +361,7 @@ func LoadConfig() {
Config.NoUpdateModTime = *noUpdateModTime
Config.BackupDir = *backupDir
Config.Suffix = *suffix
Config.BufferSize = bufferSize
ConfigPath = *configFile