From 967fc6d7f43beb508d3eb92c7a4dab6d85215f00 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 27 Aug 2023 23:10:58 +0100 Subject: [PATCH] lib/multipart: fix accounting for multipart transfers This change makes sure the accouting is done when bytes are taken out of the buffer rather than put in. See: https://forum.rclone.org/t/improve-transfer-stats-calculation-for-multipart-uploads/41172 --- fs/accounting/accounting.go | 12 ++++++++++++ lib/multipart/multipart.go | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go index 5b1046e5b..29681b604 100644 --- a/fs/accounting/accounting.go +++ b/fs/accounting/accounting.go @@ -632,3 +632,15 @@ func UnWrap(in io.Reader) (unwrapped io.Reader, wrap WrapFn) { } return acc.OldStream(), acc.WrapStream } + +// UnWrapAccounting unwraps a reader returning unwrapped and acc a +// pointer to the accounting. +// +// The caller is expected to manage the accounting at this point. +func UnWrapAccounting(in io.Reader) (unwrapped io.Reader, acc *Account) { + a, ok := in.(*accountStream) + if !ok { + return in, nil + } + return a.in, a.acc +} diff --git a/lib/multipart/multipart.go b/lib/multipart/multipart.go index a54c357eb..774076119 100644 --- a/lib/multipart/multipart.go +++ b/lib/multipart/multipart.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/pool" @@ -87,10 +88,16 @@ func UploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.Reader, opt U size = src.Size() ) + // Do the accounting manually + in, acc := accounting.UnWrapAccounting(in) + for partNum := int64(0); !finished; partNum++ { // Get a block of memory from the pool and token which limits concurrency. tokens.Get() rw := NewRW() + if acc != nil { + rw.SetAccounting(acc.AccountRead) + } free := func() { // return the memory and token