1
mirror of https://github.com/rclone/rclone synced 2025-03-21 03:04:23 +01:00

local: speed up multi thread downloads by using sparse files on Windows

Before this change rclone didn't use sparse files on Windows. This
means that when you downloaded a file with multithread download it
wrote the entire file with zeros first on the first write not at the
start of the file.

This change makes the file be sparse on Windows. Linux/macOS files
were already sparse.
This commit is contained in:
Nick Craig-Wood 2020-03-06 12:41:48 +00:00
parent dfc7215bf9
commit a492c0fb0e
4 changed files with 29 additions and 0 deletions

@ -1068,6 +1068,12 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err)
}
// Set the file to be a sparse file (important on Windows)
err = setSparse(out)
if err != nil {
fs.Debugf(o, "Failed to set sparse: %v", err)
}
return out, nil
}

@ -8,3 +8,8 @@ import "os"
func preAllocate(size int64, out *os.File) error {
return nil
}
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
return nil
}

@ -44,3 +44,8 @@ again:
// }
return err
}
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
return nil
}

@ -77,3 +77,16 @@ func preAllocate(size int64, out *os.File) error {
return nil
}
const (
FSCTL_SET_SPARSE = 0x000900c4
)
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
err := syscall.DeviceIoControl(syscall.Handle(out.Fd()), FSCTL_SET_SPARSE, nil, 0, nil, 0, nil, nil)
if err != nil {
return errors.Wrap(err, "DeviceIoControl FSCTL_SET_SPARSE")
}
return nil
}