cmdtest: end-to-end test for commands, flags and environment variables

There was no easy way to automatically test the end-to-end functionality
of commands, flags, environment variables etc.

The need for end-to-end testing was highlighted by the issues fixed
in #5341. There was no automated test to continually verify current
behaviour, nor a framework to quickly test the correctness of the fixes.

This change adds an end-to-end testing framework in the cmdtest folder.
It has some simple examples in func TestCmdTest in cmdtest_test.go. The
tests should be readable by anybody familiar with rclone and look like
this:

    // Test the rclone version command with debug logging (-vv)
    out, err = rclone("version", "-vv")
    if assert.NoError(t, err) {
        assert.Contains(t, out, "rclone v")
        assert.Contains(t, out, "os/version:")
        assert.Contains(t, out, " DEBUG : ")
    }

The end-to-end tests are executed just like the Go unit tests, that is:

    go test ./cmdtest -v

The change also contains a thorough test of environment variables in
environment_test.go.

Thanks to @ncw for encouragement and introduction to the TestMain trick.

CONTRIBUTING.md

@@ -228,6 +228,7 @@ with modules beneath.
   * cmd - the rclone commands
     * all - import this to load all the commands
     * ...commands
+  * cmdtest - end-to-end tests of commands, flags, environment variables,...
   * docs - the documentation and website
     * content - adjust these docs only - everything else is autogenerated
       * command - these are auto generated - edit the corresponding .go file

cmdtest/cmdtest.go

@@ -0,0 +1,19 @@
+// Package cmdtest creates a testable interface to rclone main
+//
+// The interface is used to perform end-to-end test of
+// commands, flags, environment variables etc.
+//
+package cmdtest
+
+// The rest of this file is a 1:1 copy from rclone.go
+
+import (
+	_ "github.com/rclone/rclone/backend/all" // import all backends
+	"github.com/rclone/rclone/cmd"
+	_ "github.com/rclone/rclone/cmd/all"    // import all commands
+	_ "github.com/rclone/rclone/lib/plugin" // import plugins
+)
+
+func main() {
+	cmd.Main()
+}

cmdtest/cmdtest_test.go

@@ -0,0 +1,228 @@
+// cmdtest_test creates a testable interface to rclone main
+//
+// The interface is used to perform end-to-end test of
+// commands, flags, environment variables etc.
+
+package cmdtest
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMain is initially called by go test to initiate the testing.
+// TestMain is also called during the tests to start rclone main in a fresh context (using exec.Command).
+// The context is determined by setting/finding the environment variable RCLONE_TEST_MAIN
+func TestMain(m *testing.M) {
+	_, found := os.LookupEnv(rcloneTestMain)
+	if !found {
+		// started by Go test => execute tests
+		err := os.Setenv(rcloneTestMain, "true")
+		if err != nil {
+			log.Fatalf("Unable to set %s: %s", rcloneTestMain, err.Error())
+		}
+		os.Exit(m.Run())
+	} else {
+		// started by func rcloneExecMain => call rclone main in cmdtest.go
+		err := os.Unsetenv(rcloneTestMain)
+		if err != nil {
+			log.Fatalf("Unable to unset %s: %s", rcloneTestMain, err.Error())
+		}
+		main()
+	}
+}
+
+const rcloneTestMain = "RCLONE_TEST_MAIN"
+
+// rcloneExecMain calls rclone with the given environment and arguments.
+// The environment variables are in a single string separated by ;
+// The terminal output is retuned as a string.
+func rcloneExecMain(env string, args ...string) (string, error) {
+	_, found := os.LookupEnv(rcloneTestMain)
+	if !found {
+		log.Fatalf("Unexpected execution path: %s is missing.", rcloneTestMain)
+	}
+	// make a call to self to execute rclone main in a predefined environment (enters TestMain above)
+	command := exec.Command(os.Args[0], args...)
+	command.Env = getEnvInitial()
+	if env != "" {
+		command.Env = append(command.Env, strings.Split(env, ";")...)
+	}
+	out, err := command.CombinedOutput()
+	return string(out), err
+}
+
+// rcloneEnv calls rclone with the given environment and arguments.
+// The environment variables are in a single string separated by ;
+// The test config file is automatically configured in RCLONE_CONFIG.
+// The terminal output is retuned as a string.
+func rcloneEnv(env string, args ...string) (string, error) {
+	envConfig := env
+	if testConfig != "" {
+		if envConfig != "" {
+			envConfig += ";"
+		}
+		envConfig += "RCLONE_CONFIG=" + testConfig
+	}
+	return rcloneExecMain(envConfig, args...)
+}
+
+// rclone calls rclone with the given arguments, E.g. "version","--help".
+// The test config file is automatically configured in RCLONE_CONFIG.
+// The terminal output is retuned as a string.
+func rclone(args ...string) (string, error) {
+	return rcloneEnv("", args...)
+}
+
+// getEnvInitial returns the os environment variables cleaned for RCLONE_ vars (except RCLONE_TEST_MAIN).
+func getEnvInitial() []string {
+	if envInitial == nil {
+		// Set initial environment variables
+		osEnv := os.Environ()
+		for i := range osEnv {
+			if !strings.HasPrefix(osEnv[i], "RCLONE_") || strings.HasPrefix(osEnv[i], rcloneTestMain) {
+				envInitial = append(envInitial, osEnv[i])
+			}
+		}
+	}
+	return envInitial
+}
+
+var envInitial []string
+
+// createTestEnvironment creates a temporary testFolder and
+// sets testConfig to testFolder/rclone.config.
+func createTestEnvironment(t *testing.T) {
+	//Set temporary folder for config and test data
+	tempFolder, err := ioutil.TempDir("", "rclone_cmdtest_")
+	require.NoError(t, err)
+	testFolder = filepath.ToSlash(tempFolder)
+
+	// Set path to temporary config file
+	testConfig = testFolder + "/rclone.config"
+}
+
+var testFolder string
+var testConfig string
+
+// removeTestEnvironment removes the test environment created by createTestEnvironment
+func removeTestEnvironment(t *testing.T) {
+	// Remove temporary folder with all contents
+	err := os.RemoveAll(testFolder)
+	require.NoError(t, err)
+}
+
+// createTestFile creates the file testFolder/name
+func createTestFile(name string, t *testing.T) string {
+	err := ioutil.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666)
+	require.NoError(t, err)
+	return testFolder + "/" + name
+}
+
+// createTestFolder creates the folder testFolder/name
+func createTestFolder(name string, t *testing.T) string {
+	err := os.Mkdir(testFolder+"/"+name, 0777)
+	require.NoError(t, err)
+	return testFolder + "/" + name
+}
+
+// createSimpleTestData creates simple test data in testFolder/subFolder
+func createSimpleTestData(t *testing.T) string {
+	createTestFolder("testdata", t)
+	createTestFile("testdata/file1.txt", t)
+	createTestFile("testdata/file2.txt", t)
+	createTestFolder("testdata/folderA", t)
+	createTestFile("testdata/folderA/fileA1.txt", t)
+	createTestFile("testdata/folderA/fileA2.txt", t)
+	createTestFolder("testdata/folderA/folderAA", t)
+	createTestFile("testdata/folderA/folderAA/fileAA1.txt", t)
+	createTestFile("testdata/folderA/folderAA/fileAA2.txt", t)
+	createTestFolder("testdata/folderB", t)
+	createTestFile("testdata/folderB/fileB1.txt", t)
+	createTestFile("testdata/folderB/fileB2.txt", t)
+	return testFolder + "/testdata"
+}
+
+// removeSimpleTestData removes the test data created by createSimpleTestData
+func removeSimpleTestData(t *testing.T) {
+	err := os.RemoveAll(testFolder + "/testdata")
+	require.NoError(t, err)
+}
+
+// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
+func TestCmdTest(t *testing.T) {
+	createTestEnvironment(t)
+	defer removeTestEnvironment(t)
+
+	// Test simple call and output from rclone
+	out, err := rclone("version")
+	t.Logf("rclone version\n" + out)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone v")
+		assert.Contains(t, out, "version: ")
+		assert.NotContains(t, out, "Error:")
+		assert.NotContains(t, out, "--help")
+		assert.NotContains(t, out, " DEBUG : ")
+		assert.Regexp(t, "rclone\\s+v\\d+\\.\\d+", out) // rclone v_.__
+	}
+
+	// Test multiple arguments and DEBUG output
+	out, err = rclone("version", "-vv")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone v")
+		assert.Contains(t, out, " DEBUG : ")
+	}
+
+	// Test error and error output
+	out, err = rclone("version", "--provoke-an-error")
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "exit status 1")
+		assert.Contains(t, out, "Error: unknown flag")
+	}
+
+	// Test effect of environment variable
+	env := "RCLONE_LOG_LEVEL=DEBUG"
+	out, err = rcloneEnv(env, "version")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone v")
+		assert.Contains(t, out, " DEBUG : ")
+	}
+
+	// Test effect of multiple environment variables, including one with ,
+	env = "RCLONE_LOG_LEVEL=DEBUG;RCLONE_LOG_FORMAT=date,shortfile;RCLONE_STATS=173ms"
+	out, err = rcloneEnv(env, "version")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone v")
+		assert.Contains(t, out, " DEBUG : ")
+		assert.Regexp(t, "[^\\s]+\\.go:\\d+:", out) // ___.go:__:
+		assert.Contains(t, out, "173ms")
+	}
+
+	// Test setup of config file
+	out, err = rclone("config", "create", "myLocal", "local")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "[myLocal]")
+		assert.Contains(t, out, "type = local")
+	}
+
+	// Test creation of simple test data
+	createSimpleTestData(t)
+	defer removeSimpleTestData(t)
+
+	// Test access to config file and simple test data
+	out, err = rclone("lsl", "myLocal:"+testFolder)
+	t.Logf("rclone lsl myLocal:testFolder\n" + out)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone.config")
+		assert.Contains(t, out, "testdata/folderA/fileA1.txt")
+	}
+
+}

cmdtest/environment_test.go

@@ -0,0 +1,276 @@
+// environment_test tests the use and precedence of environment variables
+//
+// The tests rely on functions defined in cmdtest_test.go
+
+package cmdtest
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
+func TestEnvironmentVariables(t *testing.T) {
+
+	createTestEnvironment(t)
+	defer removeTestEnvironment(t)
+
+	testdataPath := createSimpleTestData(t)
+	defer removeSimpleTestData(t)
+
+	// Non backend flags
+	// =================
+
+	// First verify default behaviour of the implicit max_depth=-1
+	env := ""
+	out, err := rcloneEnv(env, "lsl", testFolder)
+	//t.Logf("\n" + out)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "rclone.config") // depth 1
+		assert.Contains(t, out, "file1.txt")     // depth 2
+		assert.Contains(t, out, "fileA1.txt")    // depth 3
+		assert.Contains(t, out, "fileAA1.txt")   // depth 4
+	}
+
+	// Test of flag.Value
+	env = "RCLONE_MAX_DEPTH=2"
+	out, err = rcloneEnv(env, "lsl", testFolder)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "file1.txt")     // depth 2
+		assert.NotContains(t, out, "fileA1.txt") // depth 3
+	}
+
+	// Test of flag.Changed (tests #5341 Issue1)
+	env = "RCLONE_LOG_LEVEL=DEBUG"
+	out, err = rcloneEnv(env, "version", "--quiet")
+	if assert.Error(t, err) {
+		assert.Contains(t, out, " DEBUG : ")
+		assert.Contains(t, out, "Can't set -q and --log-level")
+		assert.Contains(t, "exit status 1", err.Error())
+	}
+
+	// Test of flag.DefValue
+	env = "RCLONE_STATS=173ms"
+	out, err = rcloneEnv(env, "help", "flags")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "(default 173ms)")
+	}
+
+	// Test of command line flags overriding environment flags
+	env = "RCLONE_MAX_DEPTH=2"
+	out, err = rcloneEnv(env, "lsl", testFolder, "--max-depth", "3")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "fileA1.txt")     // depth 3
+		assert.NotContains(t, out, "fileAA1.txt") // depth 4
+	}
+
+	// Test of debug logging while initialising flags from environment (tests #5241 Enhance1)
+	env = "RCLONE_STATS=173ms"
+	out, err = rcloneEnv(env, "version", "-vv")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, " DEBUG : ")
+		assert.Contains(t, out, "--stats")
+		assert.Contains(t, out, "173ms")
+		assert.Contains(t, out, "RCLONE_STATS=")
+	}
+
+	// Backend flags and option precedence
+	// ===================================
+
+	// Test approach:
+	// Verify no symlink warning when skip_links=true one the level with highest precedence
+	// and skip_links=false on all levels with lower precedence
+	//
+	// Reference: https://rclone.org/docs/#precedence
+
+	// Create a symlink in test data
+	err = os.Symlink(testdataPath+"/folderA", testdataPath+"/symlinkA")
+	if runtime.GOOS == "windows" {
+		errNote := "The policy settings on Windows often prohibit the creation of symlinks due to security issues.\n"
+		errNote += "You can safely ignore this test, if your change didn't affect environment variables."
+		require.NoError(t, err, errNote)
+	} else {
+		require.NoError(t, err)
+	}
+
+	// Create a local remote with explicit skip_links=false
+	out, err = rclone("config", "create", "myLocal", "local", "skip_links", "false")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "[myLocal]")
+		assert.Contains(t, out, "type = local")
+		assert.Contains(t, out, "skip_links = false")
+	}
+
+	// Verify symlink warning when skip_links=false on all levels
+	env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
+	out, err = rcloneEnv(env, "lsd", "myLocal,skip_links=false:"+testdataPath, "--skip-links=false")
+	//t.Logf("\n" + out)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "NOTICE: symlinkA:")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test precedence of connection strings
+	env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
+	out, err = rcloneEnv(env, "lsd", "myLocal,skip_links:"+testdataPath, "--skip-links=false")
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test precedence of command line flags
+	env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath, "--skip-links")
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test precedence of remote specific environment variables (tests #5341 Issue2)
+	env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test precedence of backend specific environment variables (tests #5341 Issue3)
+	env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test precedence of backend generic environment variables
+	env = "RCLONE_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Recreate the test remote with explicit skip_links=true
+	out, err = rclone("config", "create", "myLocal", "local", "skip_links", "true")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "[myLocal]")
+		assert.Contains(t, out, "type = local")
+		assert.Contains(t, out, "skip_links = true")
+	}
+
+	// Test precedence of config file options
+	env = ""
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Recreate the test remote with rclone defaults, that is implicit skip_links=false
+	out, err = rclone("config", "create", "myLocal", "local")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "[myLocal]")
+		assert.Contains(t, out, "type = local")
+		assert.NotContains(t, out, "skip_links")
+	}
+
+	// Verify the rclone default value (implicit skip_links=false)
+	env = ""
+	out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "NOTICE: symlinkA:")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Display of backend defaults (tests #4659)
+	//------------------------------------------
+
+	env = "RCLONE_DRIVE_CHUNK_SIZE=111M"
+	out, err = rcloneEnv(env, "help", "flags")
+	if assert.NoError(t, err) {
+		assert.Regexp(t, "--drive-chunk-size[^\\(]+\\(default 111M\\)", out)
+	}
+
+	// Options on referencing remotes (alias, crypt, etc.)
+	//----------------------------------------------------
+
+	// Create alias remote on myLocal having implicit skip_links=false
+	out, err = rclone("config", "create", "myAlias", "alias", "remote", "myLocal:"+testdataPath)
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "[myAlias]")
+		assert.Contains(t, out, "type = alias")
+		assert.Contains(t, out, "remote = myLocal:")
+	}
+
+	// Verify symlink warnings on the alias
+	env = ""
+	out, err = rcloneEnv(env, "lsd", "myAlias:")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "NOTICE: symlinkA:")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test backend generic flags
+	// having effect on the underlying local remote
+	env = "RCLONE_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myAlias:")
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test backend specific flags
+	// having effect on the underlying local remote
+	env = "RCLONE_LOCAL_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myAlias:")
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test remote specific flags
+	// having no effect unless supported by the immediate remote (alias)
+	env = "RCLONE_CONFIG_MYALIAS_SKIP_LINKS=true"
+	out, err = rcloneEnv(env, "lsd", "myAlias:")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "NOTICE: symlinkA:")
+		assert.Contains(t, out, "folderA")
+	}
+
+	env = "RCLONE_CONFIG_MYALIAS_REMOTE=" + "myLocal:" + testdataPath + "/folderA"
+	out, err = rcloneEnv(env, "lsl", "myAlias:")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "fileA1.txt")
+		assert.NotContains(t, out, "fileB1.txt")
+	}
+
+	// Test command line flags
+	// having effect on the underlying local remote
+	env = ""
+	out, err = rcloneEnv(env, "lsd", "myAlias:", "--skip-links")
+	if assert.NoError(t, err) {
+		assert.NotContains(t, out, "symlinkA")
+		assert.Contains(t, out, "folderA")
+	}
+
+	// Test connection specific flags
+	// having no effect unless supported by the immediate remote (alias)
+	env = ""
+	out, err = rcloneEnv(env, "lsd", "myAlias,skip_links:")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "NOTICE: symlinkA:")
+		assert.Contains(t, out, "folderA")
+	}
+
+	env = ""
+	out, err = rcloneEnv(env, "lsl", "myAlias,remote='myLocal:"+testdataPath+"/folderA':", "-vv")
+	if assert.NoError(t, err) {
+		assert.Contains(t, out, "fileA1.txt")
+		assert.NotContains(t, out, "fileB1.txt")
+	}
+
+}

docs/content/docs.md

@@ -239,6 +239,13 @@ However using the connection string syntax, this does work.
 
     rclone copy "gdrive,shared_with_me:shared-file.txt" gdrive:
 
+Note that the connection string only affects the options of the immediate 
+backend. If for example gdriveCrypt is a crypt based on gdrive, then the 
+following command **will not work** as intended, because 
+`shared_with_me` is ignored by the crypt backend:
+
+    rclone copy "gdriveCrypt,shared_with_me:shared-file.txt" gdriveCrypt:
+
 The connection strings have the following syntax
 
     remote,parameter=value,parameter2=value2:path/to/dir
@@ -2157,6 +2164,11 @@ mys3:
 Note that if you want to create a remote using environment variables
 you must create the `..._TYPE` variable as above.
 
+Note that you can only set the options of the immediate backend, 
+so RCLONE_CONFIG_MYS3CRYPT_ACCESS_KEY_ID has no effect, if myS3Crypt is 
+a crypt remote based on an S3 remote. However RCLONE_S3_ACCESS_KEY_ID will 
+set the access key of all remotes using S3, including myS3Crypt.
+
 Note also that now rclone has [connection strings](#connection-strings),
 it is probably easier to use those instead which makes the above example