mirror of
https://github.com/rclone/rclone
synced 2024-12-24 15:43:45 +01:00
6d0063d685
Before this change we checked the transfer was out of range only
before the Read call. This means that we returned all the data to the
reader before declaring an error. This means that some backends wrote
the file even though an error was returned.
This fix checks the transfer after the Read as well, and chops the
excess characters off the read data if we are over the limit so that
we don't ever deliver all the data.
This fixes the tests introduced as part of 6f1766dd9e and #2672
on backends other than local.
282 lines
7.1 KiB
Go
282 lines
7.1 KiB
Go
package accounting
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"strings"
|
|
"testing"
|
|
"unicode/utf8"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/asyncreader"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Check it satisfies the interfaces
|
|
var (
|
|
_ io.ReadCloser = &Account{}
|
|
_ io.Reader = &accountStream{}
|
|
_ Accounter = &Account{}
|
|
_ Accounter = &accountStream{}
|
|
)
|
|
|
|
func TestNewAccountSizeName(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 1, "test")
|
|
assert.Equal(t, in, acc.in)
|
|
assert.Equal(t, acc, stats.inProgress.get("test"))
|
|
err := acc.Close()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, acc, stats.inProgress.get("test"))
|
|
acc.Done()
|
|
assert.Nil(t, stats.inProgress.get("test"))
|
|
}
|
|
|
|
func TestAccountWithBuffer(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
|
|
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, -1, "test")
|
|
acc.WithBuffer()
|
|
// should have a buffer for an unknown size
|
|
_, ok := acc.in.(*asyncreader.AsyncReader)
|
|
require.True(t, ok)
|
|
assert.NoError(t, acc.Close())
|
|
|
|
acc = newAccountSizeName(stats, in, 1, "test")
|
|
acc.WithBuffer()
|
|
// should not have a buffer for a small size
|
|
_, ok = acc.in.(*asyncreader.AsyncReader)
|
|
require.False(t, ok)
|
|
assert.NoError(t, acc.Close())
|
|
}
|
|
|
|
func TestAccountGetUpdateReader(t *testing.T) {
|
|
test := func(doClose bool) func(t *testing.T) {
|
|
return func(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 1, "test")
|
|
|
|
assert.Equal(t, in, acc.GetReader())
|
|
assert.Equal(t, acc, stats.inProgress.get("test"))
|
|
|
|
if doClose {
|
|
// close the account before swapping it out
|
|
require.NoError(t, acc.Close())
|
|
}
|
|
|
|
in2 := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
|
|
acc.UpdateReader(in2)
|
|
|
|
assert.Equal(t, in2, acc.GetReader())
|
|
assert.Equal(t, acc, stats.inProgress.get("test"))
|
|
|
|
assert.NoError(t, acc.Close())
|
|
}
|
|
}
|
|
t.Run("NoClose", test(false))
|
|
t.Run("Close", test(true))
|
|
}
|
|
|
|
func TestAccountRead(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 1, "test")
|
|
|
|
assert.True(t, acc.start.IsZero())
|
|
assert.Equal(t, 0, acc.lpBytes)
|
|
assert.Equal(t, int64(0), acc.bytes)
|
|
assert.Equal(t, int64(0), stats.bytes)
|
|
|
|
var buf = make([]byte, 2)
|
|
n, err := acc.Read(buf)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, n)
|
|
assert.Equal(t, []byte{1, 2}, buf[:n])
|
|
|
|
assert.False(t, acc.start.IsZero())
|
|
assert.Equal(t, 2, acc.lpBytes)
|
|
assert.Equal(t, int64(2), acc.bytes)
|
|
assert.Equal(t, int64(2), stats.bytes)
|
|
|
|
n, err = acc.Read(buf)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 1, n)
|
|
assert.Equal(t, []byte{3}, buf[:n])
|
|
|
|
n, err = acc.Read(buf)
|
|
assert.Equal(t, io.EOF, err)
|
|
assert.Equal(t, 0, n)
|
|
|
|
assert.NoError(t, acc.Close())
|
|
}
|
|
|
|
func TestAccountString(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 3, "test")
|
|
|
|
// FIXME not an exhaustive test!
|
|
|
|
assert.Equal(t, "test: 0% /3, 0/s, -", strings.TrimSpace(acc.String()))
|
|
|
|
var buf = make([]byte, 2)
|
|
n, err := acc.Read(buf)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, n)
|
|
|
|
assert.Equal(t, "test: 66% /3, 0/s, -", strings.TrimSpace(acc.String()))
|
|
|
|
assert.NoError(t, acc.Close())
|
|
}
|
|
|
|
// Test the Accounter interface methods on Account and accountStream
|
|
func TestAccountAccounter(t *testing.T) {
|
|
in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 3, "test")
|
|
|
|
assert.True(t, in == acc.OldStream())
|
|
|
|
in2 := ioutil.NopCloser(bytes.NewBuffer([]byte{2, 3, 4}))
|
|
|
|
acc.SetStream(in2)
|
|
assert.True(t, in2 == acc.OldStream())
|
|
|
|
r := acc.WrapStream(in)
|
|
as, ok := r.(Accounter)
|
|
require.True(t, ok)
|
|
assert.True(t, in == as.OldStream())
|
|
assert.True(t, in2 == acc.OldStream())
|
|
accs, ok := r.(*accountStream)
|
|
require.True(t, ok)
|
|
assert.Equal(t, acc, accs.acc)
|
|
assert.True(t, in == accs.in)
|
|
|
|
// Check Read on the accountStream
|
|
var buf = make([]byte, 2)
|
|
n, err := r.Read(buf)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 2, n)
|
|
assert.Equal(t, []byte{1, 2}, buf[:n])
|
|
|
|
// Test that we can get another accountstream out
|
|
in3 := ioutil.NopCloser(bytes.NewBuffer([]byte{3, 1, 2}))
|
|
r2 := as.WrapStream(in3)
|
|
as2, ok := r2.(Accounter)
|
|
require.True(t, ok)
|
|
assert.True(t, in3 == as2.OldStream())
|
|
assert.True(t, in2 == acc.OldStream())
|
|
accs2, ok := r2.(*accountStream)
|
|
require.True(t, ok)
|
|
assert.Equal(t, acc, accs2.acc)
|
|
assert.True(t, in3 == accs2.in)
|
|
|
|
// Test we can set this new accountStream
|
|
as2.SetStream(in)
|
|
assert.True(t, in == as2.OldStream())
|
|
|
|
// Test UnWrap on accountStream
|
|
unwrapped, wrap := UnWrap(r2)
|
|
assert.True(t, unwrapped == in)
|
|
r3 := wrap(in2)
|
|
assert.True(t, in2 == r3.(Accounter).OldStream())
|
|
|
|
// TestUnWrap on a normal io.Reader
|
|
unwrapped, wrap = UnWrap(in2)
|
|
assert.True(t, unwrapped == in2)
|
|
assert.True(t, wrap(in3) == in3)
|
|
|
|
}
|
|
|
|
func TestAccountMaxTransfer(t *testing.T) {
|
|
old := fs.Config.MaxTransfer
|
|
oldMode := fs.Config.CutoffMode
|
|
|
|
fs.Config.MaxTransfer = 15
|
|
defer func() {
|
|
fs.Config.MaxTransfer = old
|
|
fs.Config.CutoffMode = oldMode
|
|
}()
|
|
|
|
in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
|
|
stats := NewStats()
|
|
acc := newAccountSizeName(stats, in, 1, "test")
|
|
|
|
var b = make([]byte, 10)
|
|
|
|
n, err := acc.Read(b)
|
|
assert.Equal(t, 10, n)
|
|
assert.NoError(t, err)
|
|
n, err = acc.Read(b)
|
|
assert.Equal(t, 5, n)
|
|
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
|
|
n, err = acc.Read(b)
|
|
assert.Equal(t, 0, n)
|
|
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
|
|
assert.True(t, fserrors.IsFatalError(err))
|
|
|
|
fs.Config.CutoffMode = fs.CutoffModeSoft
|
|
stats = NewStats()
|
|
acc = newAccountSizeName(stats, in, 1, "test")
|
|
|
|
n, err = acc.Read(b)
|
|
assert.Equal(t, 10, n)
|
|
assert.NoError(t, err)
|
|
n, err = acc.Read(b)
|
|
assert.Equal(t, 10, n)
|
|
assert.NoError(t, err)
|
|
n, err = acc.Read(b)
|
|
assert.Equal(t, 10, n)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func TestShortenName(t *testing.T) {
|
|
for _, test := range []struct {
|
|
in string
|
|
size int
|
|
want string
|
|
}{
|
|
{"", 0, ""},
|
|
{"abcde", 10, "abcde"},
|
|
{"abcde", 0, "abcde"},
|
|
{"abcde", -1, "abcde"},
|
|
{"abcde", 5, "abcde"},
|
|
{"abcde", 4, "ab…e"},
|
|
{"abcde", 3, "a…e"},
|
|
{"abcde", 2, "a…"},
|
|
{"abcde", 1, "…"},
|
|
{"abcdef", 6, "abcdef"},
|
|
{"abcdef", 5, "ab…ef"},
|
|
{"abcdef", 4, "ab…f"},
|
|
{"abcdef", 3, "a…f"},
|
|
{"abcdef", 2, "a…"},
|
|
{"áßcdèf", 1, "…"},
|
|
{"áßcdè", 5, "áßcdè"},
|
|
{"áßcdè", 4, "áß…è"},
|
|
{"áßcdè", 3, "á…è"},
|
|
{"áßcdè", 2, "á…"},
|
|
{"áßcdè", 1, "…"},
|
|
{"áßcdèł", 6, "áßcdèł"},
|
|
{"áßcdèł", 5, "áß…èł"},
|
|
{"áßcdèł", 4, "áß…ł"},
|
|
{"áßcdèł", 3, "á…ł"},
|
|
{"áßcdèł", 2, "á…"},
|
|
{"áßcdèł", 1, "…"},
|
|
} {
|
|
t.Run(fmt.Sprintf("in=%q, size=%d", test.in, test.size), func(t *testing.T) {
|
|
got := shortenName(test.in, test.size)
|
|
assert.Equal(t, test.want, got)
|
|
if test.size > 0 {
|
|
assert.True(t, utf8.RuneCountInString(got) <= test.size, "too big")
|
|
}
|
|
})
|
|
}
|
|
}
|