rclone/backend/chunker/chunker_internal_test.go

package chunker

import (
	"context"
	"flag"
	"fmt"
	"testing"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/fstest/fstests"
	"github.com/stretchr/testify/assert"
)

// Command line flags
var (
	UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
)

// test that chunking does not break large uploads
func (f *Fs) InternalTestPutLarge(t *testing.T, kilobytes int) {
	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
			Size:    int64(kilobytes) * int64(fs.KibiByte),
		})
	})
}

func (f *Fs) InternalTestChunkNameFormat(t *testing.T) {
	savedNameFormat := f.opt.NameFormat
	savedStartFrom := f.opt.StartFrom
	defer func() {
		// restore original settings
		_ = f.parseNameFormat(savedNameFormat)
		f.opt.StartFrom = savedStartFrom
	}()
	var err error

	err = f.parseNameFormat("*.rclone_chunk.###")
	assert.NoError(t, err)
	assert.Equal(t, `%s.rclone_chunk.%03d`, f.nameFormat)
	assert.Equal(t, `^(.+)\.rclone_chunk\.([0-9]{3,})$`, f.nameRegexp.String())

	err = f.parseNameFormat("*.rclone_chunk.#")
	assert.NoError(t, err)
	assert.Equal(t, `%s.rclone_chunk.%d`, f.nameFormat)
	assert.Equal(t, `^(.+)\.rclone_chunk\.([0-9]+)$`, f.nameRegexp.String())

	err = f.parseNameFormat("*_chunk_#####")
	assert.NoError(t, err)
	assert.Equal(t, `%s_chunk_%05d`, f.nameFormat)
	assert.Equal(t, `^(.+)_chunk_([0-9]{5,})$`, f.nameRegexp.String())

	err = f.parseNameFormat("*-chunk-#")
	assert.NoError(t, err)
	assert.Equal(t, `%s-chunk-%d`, f.nameFormat)
	assert.Equal(t, `^(.+)-chunk-([0-9]+)$`, f.nameRegexp.String())

	err = f.parseNameFormat("_*-chunk-##,")
	assert.NoError(t, err)
	assert.Equal(t, `_%s-chunk-%02d,`, f.nameFormat)
	assert.Equal(t, `^_(.+)-chunk-([0-9]{2,}),$`, f.nameRegexp.String())

	err = f.parseNameFormat(`*-chunk-#-%^$()[]{}.+-!?:\/`)
	assert.NoError(t, err)
	assert.Equal(t, `%s-chunk-%d-%%^$()[]{}.+-!?:\/`, f.nameFormat)
	assert.Equal(t, `^(.+)-chunk-([0-9]+)-%\^\$\(\)\[\]\{\}\.\+-!\?:\\/$`, f.nameRegexp.String())

	err = f.parseNameFormat("chunk-#")
	assert.Error(t, err)

	err = f.parseNameFormat("*-chunk")
	assert.Error(t, err)

	err = f.parseNameFormat("*-*-chunk-#")
	assert.Error(t, err)

	err = f.parseNameFormat("*-chunk-#-#")
	assert.Error(t, err)

	err = f.parseNameFormat("#-chunk-*")
	assert.Error(t, err)

	err = f.parseNameFormat("*#")
	assert.NoError(t, err)

	err = f.parseNameFormat("**#")
	assert.Error(t, err)
	err = f.parseNameFormat("#*")
	assert.Error(t, err)
	err = f.parseNameFormat("")
	assert.Error(t, err)
	err = f.parseNameFormat("-")
	assert.Error(t, err)

	f.opt.StartFrom = 2
	err = f.parseNameFormat("*.chunk.###")
	assert.NoError(t, err)
	assert.Equal(t, `%s.chunk.%03d`, f.nameFormat)
	assert.Equal(t, `^(.+)\.chunk\.([0-9]{3,})$`, f.nameRegexp.String())

	assert.Equal(t, "fish.chunk.003", f.makeChunkName("fish", 1, -1))
	assert.Equal(t, "fish.chunk.011..tmp_0000054321", f.makeChunkName("fish", 9, 54321))
	assert.Equal(t, "fish.chunk.011..tmp_1234567890", f.makeChunkName("fish", 9, 1234567890))
	assert.Equal(t, "fish.chunk.1916..tmp_123456789012345", f.makeChunkName("fish", 1914, 123456789012345))

	name, chunkNo, tempNo := f.parseChunkName("fish.chunk.003")
	assert.True(t, name == "fish" && chunkNo == 1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.004..tmp_0000000021")
	assert.True(t, name == "fish" && chunkNo == 2 && tempNo == 21)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.021")
	assert.True(t, name == "fish" && chunkNo == 19 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.323..tmp_1234567890123456789")
	assert.True(t, name == "fish" && chunkNo == 321 && tempNo == 1234567890123456789)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.3")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.001")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.21")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.-21")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.004.tmp_0000000021")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_123456789")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_012345678901234567890123456789")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
	name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_-1")
	assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
}

func (f *Fs) InternalTest(t *testing.T) {
	t.Run("PutLarge", func(t *testing.T) {
		if *UploadKilobytes <= 0 {
			t.Skip("-upload-kilobytes is not set")
		}
		f.InternalTestPutLarge(t, *UploadKilobytes)
	})
	t.Run("ChunkNameFormat", func(t *testing.T) {
		f.InternalTestChunkNameFormat(t)
	})
}

var _ fstests.InternalTester = (*Fs)(nil)