fstest: factor chunked streaming tests from b2 and use in all backends

backend/b2/b2_internal_test.go

@@ -1,19 +1,11 @@
 package b2
 
 import (
-	"bytes"
-	"context"
-	"fmt"
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // Test b2 string encoding
@@ -178,56 +170,9 @@ func TestParseTimeString(t *testing.T) {
 
 }
 
-// The integration tests do a reasonable job of testing the normal
-// streaming upload but don't test the chunked streaming upload.
-func (f *Fs) InternalTestChunkedStreamingUpload(t *testing.T, size int) {
-	ctx := context.Background()
-	contents := random.String(size)
-	item := fstest.NewItem(fmt.Sprintf("chunked-streaming-upload-%d", size), contents, fstest.Time("2001-05-06T04:05:06.499Z"))
-
-	// Set chunk size to mininum value so we make chunks
-	origOpt := f.opt
-	f.opt.ChunkSize = minChunkSize
-	f.opt.UploadCutoff = 0
-	defer func() {
-		f.opt = origOpt
-	}()
-
-	// Do the streaming upload
-	src := object.NewStaticObjectInfo(item.Path, item.ModTime, -1, true, item.Hashes, f)
-	in := bytes.NewBufferString(contents)
-	dst, err := f.PutStream(ctx, in, src)
-	require.NoError(t, err)
-	defer func() {
-		assert.NoError(t, dst.Remove(ctx))
-	}()
-
-	// Check size
-	assert.Equal(t, int64(size), dst.Size())
-
-	// Check modtime
-	srcModTime := src.ModTime(ctx)
-	dstModTime := dst.ModTime(ctx)
-	assert.Equal(t, srcModTime, dstModTime)
-
-	// Make sure contents are correct
-	gotContents := fstests.ReadObject(ctx, t, dst, -1)
-	assert.Equal(t, contents, gotContents, "Contents incorrect")
-}
-
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
-	for _, size := range []fs.SizeSuffix{
-		minChunkSize - 1,
-		minChunkSize,
-		minChunkSize + 1,
-		(3 * minChunkSize) / 2,
-		(5 * minChunkSize) / 2,
-	} {
-		t.Run(fmt.Sprintf("ChunkedStreamingUpload/%d", size), func(t *testing.T) {
-			f.InternalTestChunkedStreamingUpload(t, int(size))
-		})
-	}
+	// Internal tests go here
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

fstest/fstests/fstests.go

@@ -230,8 +230,10 @@ func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.It
 	return contents, PutTestContentsMetadata(ctx, t, f, file, contents, true, mimeType, metadata)
 }
 
-// TestPutLarge puts file to the remote, checks it and removes it on success.
-func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
+// testPutLarge puts file to the remote, checks it and removes it on success.
+//
+// If stream is set, then it uploads the file with size -1
+func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, stream bool) {
 	var (
 		err        error
 		obj        fs.Object
@@ -242,7 +244,11 @@ func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item)
 		uploadHash = hash.NewMultiHasher()
 		in := io.TeeReader(r, uploadHash)
 
-		obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
+		size := file.Size
+		if stream {
+			size = -1
+		}
+		obji := object.NewStaticObjectInfo(file.Path, file.ModTime, size, true, nil, nil)
 		obj, err = f.Put(ctx, in, obji)
 		if file.Size == 0 && err == fs.ErrorCantUploadEmptyFiles {
 			t.Skip("Can't upload zero length files")
@@ -270,6 +276,16 @@ func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item)
 	require.NoError(t, obj.Remove(ctx))
 }
 
+// TestPutLarge puts file to the remote, checks it and removes it on success.
+func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
+	testPutLarge(ctx, t, f, file, false)
+}
+
+// TestPutLargeStreamed puts file of unknown size to the remote, checks it and removes it on success.
+func TestPutLargeStreamed(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
+	testPutLarge(ctx, t, f, file, true)
+}
+
 // ReadObject reads the contents of an object as a string
 func ReadObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string {
 	what := fmt.Sprintf("readObject(%q) limit=%d, options=%+v", obj, limit, options)
@@ -2097,6 +2113,16 @@ func Run(t *testing.T, opt *Opt) {
 								Path:    fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()),
 								Size:    int64(fileSize),
 							})
+							t.Run("Streamed", func(t *testing.T) {
+								if f.Features().PutStream == nil {
+									t.Skip("FS has no PutStream interface")
+								}
+								TestPutLargeStreamed(ctx, t, f, &fstest.Item{
+									ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+									Path:    fmt.Sprintf("chunked-%s-%s-streamed.bin", cs.String(), fileSize.String()),
+									Size:    int64(fileSize),
+								})
+							})
 						})
 					}
 				})