mirror of https://github.com/rclone/rclone
37 lines
1.6 KiB
Go
37 lines
1.6 KiB
Go
// Package chunksize calculates a suitable chunk size for large uploads
|
|
package chunksize
|
|
|
|
import (
|
|
"github.com/rclone/rclone/fs"
|
|
)
|
|
|
|
/*
|
|
Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
|
|
|
|
For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
|
|
possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
|
|
that some backends may only allow integer type parameters when specifying the chunk size.
|
|
|
|
Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
|
|
smallest chunk size necessary to allow the upload to proceed.
|
|
*/
|
|
func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
|
|
fileSize := fs.SizeSuffix(objInfo.Size())
|
|
requiredChunks := fileSize / defaultChunkSize
|
|
if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
|
|
return defaultChunkSize
|
|
}
|
|
|
|
minChunk := fileSize / fs.SizeSuffix(maxParts)
|
|
remainder := minChunk % fs.Mebi
|
|
if remainder != 0 {
|
|
minChunk += fs.Mebi - remainder
|
|
}
|
|
if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
|
|
minChunk += fs.Mebi
|
|
}
|
|
|
|
fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
|
|
return minChunk
|
|
}
|