package operations_test import ( "context" "crypto/rand" "errors" "fmt" "strings" "testing" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTruncateString(t *testing.T) { for _, test := range []struct { in string n int want string }{ { in: "", n: 0, want: "", }, { in: "Hello World", n: 5, want: "Hello", }, { in: "ááááá", n: 5, want: "áá", }, { in: "ááááá\xFF\xFF", n: 5, want: "áá\xc3", }, { in: "世世世世世", n: 7, want: "世世", }, { in: "🙂🙂🙂🙂🙂", n: 16, want: "🙂🙂🙂🙂", }, { in: "🙂🙂🙂🙂🙂", n: 15, want: "🙂🙂🙂", }, { in: "🙂🙂🙂🙂🙂", n: 14, want: "🙂🙂🙂", }, { in: "🙂🙂🙂🙂🙂", n: 13, want: "🙂🙂🙂", }, { in: "🙂🙂🙂🙂🙂", n: 12, want: "🙂🙂🙂", }, { in: "🙂🙂🙂🙂🙂", n: 11, want: "🙂🙂", }, { in: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", n: 100, want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ", }, { in: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", n: 100, want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ", }, { in: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", n: 100, want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ", }, { in: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", n: 100, want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ", }, { in: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", n: 100, want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ", }, } { got := operations.TruncateString(test.in, test.n) assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in)) assert.LessOrEqual(t, len(got), test.n) assert.GreaterOrEqual(t, len(got), test.n-3) } } func TestCopyFile(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file2 := file1 file2.Path = "sub/file2" err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) } func TestCopyFileBackupDir(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if !operations.CanServerSideMove(r.Fremote) { t.Skip("Skipping test as remote does not support server-side move or copy") } ci.BackupDir = r.FremoteName + "/backup" file1 := r.WriteFile("dst/file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1) r.CheckRemoteItems(t, file1old) err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) file1old.Path = "backup/dst/file1" r.CheckRemoteItems(t, file1old, file1) } // Test with CompareDest set func TestCopyFileCompareDest(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) ci.CompareDest = []string{r.FremoteName + "/CompareDest"} fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty compare file1 := r.WriteFile("one", "one", t1) r.CheckLocalItems(t, file1) err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) file1dst := file1 file1dst.Path = "dst/one" r.CheckRemoteItems(t, file1dst) // check old dest, empty compare file1b := r.WriteFile("one", "onet2", t2) r.CheckRemoteItems(t, file1dst) r.CheckLocalItems(t, file1b) err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path) require.NoError(t, err) file1bdst := file1b file1bdst.Path = "dst/one" r.CheckRemoteItems(t, file1bdst) // check old dest, new compare file3 := r.WriteObject(ctx, "dst/one", "one", t1) file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2) file1c := r.WriteFile("one", "onet2", t2) r.CheckRemoteItems(t, file2, file3) r.CheckLocalItems(t, file1c) err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path) require.NoError(t, err) r.CheckRemoteItems(t, file2, file3) // check empty dest, new compare file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2) file5 := r.WriteFile("two", "two", t2) r.CheckRemoteItems(t, file2, file3, file4) r.CheckLocalItems(t, file1c, file5) err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) require.NoError(t, err) r.CheckRemoteItems(t, file2, file3, file4) // check new dest, new compare err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) require.NoError(t, err) r.CheckRemoteItems(t, file2, file3, file4) // check empty dest, old compare file5b := r.WriteFile("two", "twot3", t3) r.CheckRemoteItems(t, file2, file3, file4) r.CheckLocalItems(t, file1c, file5b) err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path) require.NoError(t, err) file5bdst := file5b file5bdst.Path = "dst/two" r.CheckRemoteItems(t, file2, file3, file4, file5bdst) } // Test with CopyDest set func TestCopyFileCopyDest(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if r.Fremote.Features().Copy == nil { t.Skip("Skipping test as remote does not support server-side copy") } ci.CopyDest = []string{r.FremoteName + "/CopyDest"} fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") require.NoError(t, err) // check empty dest, empty copy file1 := r.WriteFile("one", "one", t1) r.CheckLocalItems(t, file1) err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) file1dst := file1 file1dst.Path = "dst/one" r.CheckRemoteItems(t, file1dst) // check old dest, empty copy file1b := r.WriteFile("one", "onet2", t2) r.CheckRemoteItems(t, file1dst) r.CheckLocalItems(t, file1b) err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path) require.NoError(t, err) file1bdst := file1b file1bdst.Path = "dst/one" r.CheckRemoteItems(t, file1bdst) // check old dest, new copy, backup-dir ci.BackupDir = r.FremoteName + "/BackupDir" file3 := r.WriteObject(ctx, "dst/one", "one", t1) file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2) file1c := r.WriteFile("one", "onet2", t2) r.CheckRemoteItems(t, file2, file3) r.CheckLocalItems(t, file1c) err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path) require.NoError(t, err) file2dst := file2 file2dst.Path = "dst/one" file3.Path = "BackupDir/one" r.CheckRemoteItems(t, file2, file2dst, file3) ci.BackupDir = "" // check empty dest, new copy file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2) file5 := r.WriteFile("two", "two", t2) r.CheckRemoteItems(t, file2, file2dst, file3, file4) r.CheckLocalItems(t, file1c, file5) err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) require.NoError(t, err) file4dst := file4 file4dst.Path = "dst/two" r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) // check new dest, new copy err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) require.NoError(t, err) r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) // check empty dest, old copy file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2) file7 := r.WriteFile("three", "threet3", t3) r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6) r.CheckLocalItems(t, file1c, file5, file7) err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path) require.NoError(t, err) file7dst := file7 file7dst.Path = "dst/three" r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst) } func TestCopyInplace(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if !r.Fremote.Features().PartialUploads { t.Skip("Partial uploads not supported") } ci.Inplace = true file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file2 := file1 file2.Path = "sub/file2" err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) } func TestCopyLongFileName(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) if !r.Fremote.Features().PartialUploads { t.Skip("Partial uploads not supported") } ci.Inplace = false // the default file1 := r.WriteFile("file1", "file1 contents", t1) r.CheckLocalItems(t, file1) file2 := file1 file2.Path = "sub/" + strings.Repeat("file2", 30) err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) require.NoError(t, err) r.CheckLocalItems(t, file1) r.CheckRemoteItems(t, file2) } func TestCopyFileMaxTransfer(t *testing.T) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) r := fstest.NewRun(t) defer accounting.Stats(ctx).ResetCounters() const sizeCutoff = 2048 // Make random incompressible data randomData := make([]byte, sizeCutoff) _, err := rand.Read(randomData) require.NoError(t, err) randomString := string(randomData) file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1) file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2) file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2) file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2) // Cutoff mode: Hard ci.MaxTransfer = sizeCutoff ci.CutoffMode = fs.CutoffModeHard // file1: Show a small file gets transferred OK accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) require.NoError(t, err) r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1) // file2: show a large file does not get transferred accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path) require.NotNil(t, err, "Did not get expected max transfer limit error") if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) { t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal") // Sometimes the backends or their SDKs don't pass the // error through properly, so check that it at least // has the text we expect in. assert.Contains(t, err.Error(), "max transfer limit reached") } r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1) // Cutoff mode: Cautious ci.CutoffMode = fs.CutoffModeCautious // file3: show a large file does not get transferred accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path) require.NotNil(t, err) assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful)) r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1) if isChunker(r.Fremote) { t.Log("skipping remainder of test for chunker as it involves multiple transfers") return } // Cutoff mode: Soft ci.CutoffMode = fs.CutoffModeSoft // file4: show a large file does get transferred this time accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path) require.NoError(t, err) r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1, file4) }