mirror of
https://github.com/rclone/rclone
synced 2024-12-01 10:31:57 +01:00
e64be7652a
Before this change, when not using --inplace, rclone could generate invalid file names when truncating file names to fit within the character size limits. This fixes it by taking care to truncate on UTF-8 character boundaries. See: https://forum.rclone.org/t/ssh-fx-failure-when-copying-file-with-nonstandard-characters-to-sftp-remote-with-ntfs-drive/42560/
456 lines
13 KiB
Go
456 lines
13 KiB
Go
package operations_test
|
||
|
||
import (
|
||
"context"
|
||
"crypto/rand"
|
||
"errors"
|
||
"fmt"
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/rclone/rclone/fs"
|
||
"github.com/rclone/rclone/fs/accounting"
|
||
"github.com/rclone/rclone/fs/operations"
|
||
"github.com/rclone/rclone/fstest"
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
)
|
||
|
||
func TestTruncateString(t *testing.T) {
|
||
for _, test := range []struct {
|
||
in string
|
||
n int
|
||
want string
|
||
}{
|
||
{
|
||
in: "",
|
||
n: 0,
|
||
want: "",
|
||
}, {
|
||
in: "Hello World",
|
||
n: 5,
|
||
want: "Hello",
|
||
}, {
|
||
in: "ááááá",
|
||
n: 5,
|
||
want: "áá",
|
||
}, {
|
||
in: "ááááá\xFF\xFF",
|
||
n: 5,
|
||
want: "áá\xc3",
|
||
}, {
|
||
in: "世世世世世",
|
||
n: 7,
|
||
want: "世世",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 16,
|
||
want: "🙂🙂🙂🙂",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 15,
|
||
want: "🙂🙂🙂",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 14,
|
||
want: "🙂🙂🙂",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 13,
|
||
want: "🙂🙂🙂",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 12,
|
||
want: "🙂🙂🙂",
|
||
}, {
|
||
in: "🙂🙂🙂🙂🙂",
|
||
n: 11,
|
||
want: "🙂🙂",
|
||
}, {
|
||
in: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
|
||
n: 100,
|
||
want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
|
||
}, {
|
||
in: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
|
||
n: 100,
|
||
want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
|
||
}, {
|
||
in: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
|
||
n: 100,
|
||
want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
|
||
}, {
|
||
in: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
|
||
n: 100,
|
||
want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
|
||
}, {
|
||
in: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
|
||
n: 100,
|
||
want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ",
|
||
},
|
||
} {
|
||
got := operations.TruncateString(test.in, test.n)
|
||
assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in))
|
||
assert.LessOrEqual(t, len(got), test.n)
|
||
assert.GreaterOrEqual(t, len(got), test.n-3)
|
||
}
|
||
}
|
||
|
||
func TestCopyFile(t *testing.T) {
|
||
ctx := context.Background()
|
||
r := fstest.NewRun(t)
|
||
|
||
file1 := r.WriteFile("file1", "file1 contents", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
file2 := file1
|
||
file2.Path = "sub/file2"
|
||
|
||
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
}
|
||
|
||
func TestCopyFileBackupDir(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
if !operations.CanServerSideMove(r.Fremote) {
|
||
t.Skip("Skipping test as remote does not support server-side move or copy")
|
||
}
|
||
|
||
ci.BackupDir = r.FremoteName + "/backup"
|
||
|
||
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1)
|
||
r.CheckRemoteItems(t, file1old)
|
||
|
||
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
file1old.Path = "backup/dst/file1"
|
||
r.CheckRemoteItems(t, file1old, file1)
|
||
}
|
||
|
||
// Test with CompareDest set
|
||
func TestCopyFileCompareDest(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
|
||
ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
|
||
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
|
||
require.NoError(t, err)
|
||
|
||
// check empty dest, empty compare
|
||
file1 := r.WriteFile("one", "one", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
|
||
file1dst := file1
|
||
file1dst.Path = "dst/one"
|
||
|
||
r.CheckRemoteItems(t, file1dst)
|
||
|
||
// check old dest, empty compare
|
||
file1b := r.WriteFile("one", "onet2", t2)
|
||
r.CheckRemoteItems(t, file1dst)
|
||
r.CheckLocalItems(t, file1b)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
|
||
require.NoError(t, err)
|
||
|
||
file1bdst := file1b
|
||
file1bdst.Path = "dst/one"
|
||
|
||
r.CheckRemoteItems(t, file1bdst)
|
||
|
||
// check old dest, new compare
|
||
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
|
||
file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2)
|
||
file1c := r.WriteFile("one", "onet2", t2)
|
||
r.CheckRemoteItems(t, file2, file3)
|
||
r.CheckLocalItems(t, file1c)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
|
||
require.NoError(t, err)
|
||
|
||
r.CheckRemoteItems(t, file2, file3)
|
||
|
||
// check empty dest, new compare
|
||
file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2)
|
||
file5 := r.WriteFile("two", "two", t2)
|
||
r.CheckRemoteItems(t, file2, file3, file4)
|
||
r.CheckLocalItems(t, file1c, file5)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
|
||
require.NoError(t, err)
|
||
|
||
r.CheckRemoteItems(t, file2, file3, file4)
|
||
|
||
// check new dest, new compare
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
|
||
require.NoError(t, err)
|
||
|
||
r.CheckRemoteItems(t, file2, file3, file4)
|
||
|
||
// check empty dest, old compare
|
||
file5b := r.WriteFile("two", "twot3", t3)
|
||
r.CheckRemoteItems(t, file2, file3, file4)
|
||
r.CheckLocalItems(t, file1c, file5b)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path)
|
||
require.NoError(t, err)
|
||
|
||
file5bdst := file5b
|
||
file5bdst.Path = "dst/two"
|
||
|
||
r.CheckRemoteItems(t, file2, file3, file4, file5bdst)
|
||
}
|
||
|
||
// Test with CopyDest set
|
||
func TestCopyFileCopyDest(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
|
||
if r.Fremote.Features().Copy == nil {
|
||
t.Skip("Skipping test as remote does not support server-side copy")
|
||
}
|
||
|
||
ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
|
||
|
||
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
|
||
require.NoError(t, err)
|
||
|
||
// check empty dest, empty copy
|
||
file1 := r.WriteFile("one", "one", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
|
||
file1dst := file1
|
||
file1dst.Path = "dst/one"
|
||
|
||
r.CheckRemoteItems(t, file1dst)
|
||
|
||
// check old dest, empty copy
|
||
file1b := r.WriteFile("one", "onet2", t2)
|
||
r.CheckRemoteItems(t, file1dst)
|
||
r.CheckLocalItems(t, file1b)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
|
||
require.NoError(t, err)
|
||
|
||
file1bdst := file1b
|
||
file1bdst.Path = "dst/one"
|
||
|
||
r.CheckRemoteItems(t, file1bdst)
|
||
|
||
// check old dest, new copy, backup-dir
|
||
|
||
ci.BackupDir = r.FremoteName + "/BackupDir"
|
||
|
||
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
|
||
file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
|
||
file1c := r.WriteFile("one", "onet2", t2)
|
||
r.CheckRemoteItems(t, file2, file3)
|
||
r.CheckLocalItems(t, file1c)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
|
||
require.NoError(t, err)
|
||
|
||
file2dst := file2
|
||
file2dst.Path = "dst/one"
|
||
file3.Path = "BackupDir/one"
|
||
|
||
r.CheckRemoteItems(t, file2, file2dst, file3)
|
||
ci.BackupDir = ""
|
||
|
||
// check empty dest, new copy
|
||
file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
|
||
file5 := r.WriteFile("two", "two", t2)
|
||
r.CheckRemoteItems(t, file2, file2dst, file3, file4)
|
||
r.CheckLocalItems(t, file1c, file5)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
|
||
require.NoError(t, err)
|
||
|
||
file4dst := file4
|
||
file4dst.Path = "dst/two"
|
||
|
||
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
|
||
|
||
// check new dest, new copy
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
|
||
require.NoError(t, err)
|
||
|
||
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
|
||
|
||
// check empty dest, old copy
|
||
file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2)
|
||
file7 := r.WriteFile("three", "threet3", t3)
|
||
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6)
|
||
r.CheckLocalItems(t, file1c, file5, file7)
|
||
|
||
err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path)
|
||
require.NoError(t, err)
|
||
|
||
file7dst := file7
|
||
file7dst.Path = "dst/three"
|
||
|
||
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst)
|
||
}
|
||
|
||
func TestCopyInplace(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
|
||
if !r.Fremote.Features().PartialUploads {
|
||
t.Skip("Partial uploads not supported")
|
||
}
|
||
|
||
ci.Inplace = true
|
||
|
||
file1 := r.WriteFile("file1", "file1 contents", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
file2 := file1
|
||
file2.Path = "sub/file2"
|
||
|
||
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
}
|
||
|
||
func TestCopyLongFileName(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
|
||
if !r.Fremote.Features().PartialUploads {
|
||
t.Skip("Partial uploads not supported")
|
||
}
|
||
|
||
ci.Inplace = false // the default
|
||
|
||
file1 := r.WriteFile("file1", "file1 contents", t1)
|
||
r.CheckLocalItems(t, file1)
|
||
|
||
file2 := file1
|
||
file2.Path = "sub/" + strings.Repeat("file2", 30)
|
||
|
||
err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1)
|
||
r.CheckRemoteItems(t, file2)
|
||
}
|
||
|
||
func TestCopyFileMaxTransfer(t *testing.T) {
|
||
ctx := context.Background()
|
||
ctx, ci := fs.AddConfig(ctx)
|
||
r := fstest.NewRun(t)
|
||
defer accounting.Stats(ctx).ResetCounters()
|
||
|
||
const sizeCutoff = 2048
|
||
|
||
// Make random incompressible data
|
||
randomData := make([]byte, sizeCutoff)
|
||
_, err := rand.Read(randomData)
|
||
require.NoError(t, err)
|
||
randomString := string(randomData)
|
||
|
||
file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1)
|
||
file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2)
|
||
file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2)
|
||
file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2)
|
||
|
||
// Cutoff mode: Hard
|
||
ci.MaxTransfer = sizeCutoff
|
||
ci.CutoffMode = fs.CutoffModeHard
|
||
|
||
// file1: Show a small file gets transferred OK
|
||
accounting.Stats(ctx).ResetCounters()
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1, file2, file3, file4)
|
||
r.CheckRemoteItems(t, file1)
|
||
|
||
// file2: show a large file does not get transferred
|
||
accounting.Stats(ctx).ResetCounters()
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path)
|
||
require.NotNil(t, err, "Did not get expected max transfer limit error")
|
||
if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) {
|
||
t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal")
|
||
// Sometimes the backends or their SDKs don't pass the
|
||
// error through properly, so check that it at least
|
||
// has the text we expect in.
|
||
assert.Contains(t, err.Error(), "max transfer limit reached")
|
||
}
|
||
r.CheckLocalItems(t, file1, file2, file3, file4)
|
||
r.CheckRemoteItems(t, file1)
|
||
|
||
// Cutoff mode: Cautious
|
||
ci.CutoffMode = fs.CutoffModeCautious
|
||
|
||
// file3: show a large file does not get transferred
|
||
accounting.Stats(ctx).ResetCounters()
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path)
|
||
require.NotNil(t, err)
|
||
assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful))
|
||
r.CheckLocalItems(t, file1, file2, file3, file4)
|
||
r.CheckRemoteItems(t, file1)
|
||
|
||
if isChunker(r.Fremote) {
|
||
t.Log("skipping remainder of test for chunker as it involves multiple transfers")
|
||
return
|
||
}
|
||
|
||
// Cutoff mode: Soft
|
||
ci.CutoffMode = fs.CutoffModeSoft
|
||
|
||
// file4: show a large file does get transferred this time
|
||
accounting.Stats(ctx).ResetCounters()
|
||
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path)
|
||
require.NoError(t, err)
|
||
r.CheckLocalItems(t, file1, file2, file3, file4)
|
||
r.CheckRemoteItems(t, file1, file4)
|
||
}
|