cmdtest: end-to-end test for commands, flags and environment variables

There was no easy way to automatically test the end-to-end functionality
of commands, flags, environment variables etc.

The need for end-to-end testing was highlighted by the issues fixed
in #5341. There was no automated test to continually verify current
behaviour, nor a framework to quickly test the correctness of the fixes.

This change adds an end-to-end testing framework in the cmdtest folder.
It has some simple examples in func TestCmdTest in cmdtest_test.go. The
tests should be readable by anybody familiar with rclone and look like
this:

    // Test the rclone version command with debug logging (-vv)
    out, err = rclone("version", "-vv")
    if assert.NoError(t, err) {
        assert.Contains(t, out, "rclone v")
        assert.Contains(t, out, "os/version:")
        assert.Contains(t, out, " DEBUG : ")
    }

The end-to-end tests are executed just like the Go unit tests, that is:

    go test ./cmdtest -v

The change also contains a thorough test of environment variables in
environment_test.go.

Thanks to @ncw for encouragement and introduction to the TestMain trick.
This commit is contained in:
Ole Frost 2021-06-04 10:56:40 +02:00 committed by Nick Craig-Wood
parent 58c99427b3
commit 5e91b93e59
5 changed files with 536 additions and 0 deletions

View File

@ -228,6 +228,7 @@ with modules beneath.
* cmd - the rclone commands
* all - import this to load all the commands
* ...commands
* cmdtest - end-to-end tests of commands, flags, environment variables,...
* docs - the documentation and website
* content - adjust these docs only - everything else is autogenerated
* command - these are auto generated - edit the corresponding .go file

19
cmdtest/cmdtest.go Normal file
View File

@ -0,0 +1,19 @@
// Package cmdtest creates a testable interface to rclone main
//
// The interface is used to perform end-to-end test of
// commands, flags, environment variables etc.
//
package cmdtest
// The rest of this file is a 1:1 copy from rclone.go
import (
_ "github.com/rclone/rclone/backend/all" // import all backends
"github.com/rclone/rclone/cmd"
_ "github.com/rclone/rclone/cmd/all" // import all commands
_ "github.com/rclone/rclone/lib/plugin" // import plugins
)
func main() {
cmd.Main()
}

228
cmdtest/cmdtest_test.go Normal file
View File

@ -0,0 +1,228 @@
// cmdtest_test creates a testable interface to rclone main
//
// The interface is used to perform end-to-end test of
// commands, flags, environment variables etc.
package cmdtest
import (
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestMain is initially called by go test to initiate the testing.
// TestMain is also called during the tests to start rclone main in a fresh context (using exec.Command).
// The context is determined by setting/finding the environment variable RCLONE_TEST_MAIN
func TestMain(m *testing.M) {
_, found := os.LookupEnv(rcloneTestMain)
if !found {
// started by Go test => execute tests
err := os.Setenv(rcloneTestMain, "true")
if err != nil {
log.Fatalf("Unable to set %s: %s", rcloneTestMain, err.Error())
}
os.Exit(m.Run())
} else {
// started by func rcloneExecMain => call rclone main in cmdtest.go
err := os.Unsetenv(rcloneTestMain)
if err != nil {
log.Fatalf("Unable to unset %s: %s", rcloneTestMain, err.Error())
}
main()
}
}
const rcloneTestMain = "RCLONE_TEST_MAIN"
// rcloneExecMain calls rclone with the given environment and arguments.
// The environment variables are in a single string separated by ;
// The terminal output is retuned as a string.
func rcloneExecMain(env string, args ...string) (string, error) {
_, found := os.LookupEnv(rcloneTestMain)
if !found {
log.Fatalf("Unexpected execution path: %s is missing.", rcloneTestMain)
}
// make a call to self to execute rclone main in a predefined environment (enters TestMain above)
command := exec.Command(os.Args[0], args...)
command.Env = getEnvInitial()
if env != "" {
command.Env = append(command.Env, strings.Split(env, ";")...)
}
out, err := command.CombinedOutput()
return string(out), err
}
// rcloneEnv calls rclone with the given environment and arguments.
// The environment variables are in a single string separated by ;
// The test config file is automatically configured in RCLONE_CONFIG.
// The terminal output is retuned as a string.
func rcloneEnv(env string, args ...string) (string, error) {
envConfig := env
if testConfig != "" {
if envConfig != "" {
envConfig += ";"
}
envConfig += "RCLONE_CONFIG=" + testConfig
}
return rcloneExecMain(envConfig, args...)
}
// rclone calls rclone with the given arguments, E.g. "version","--help".
// The test config file is automatically configured in RCLONE_CONFIG.
// The terminal output is retuned as a string.
func rclone(args ...string) (string, error) {
return rcloneEnv("", args...)
}
// getEnvInitial returns the os environment variables cleaned for RCLONE_ vars (except RCLONE_TEST_MAIN).
func getEnvInitial() []string {
if envInitial == nil {
// Set initial environment variables
osEnv := os.Environ()
for i := range osEnv {
if !strings.HasPrefix(osEnv[i], "RCLONE_") || strings.HasPrefix(osEnv[i], rcloneTestMain) {
envInitial = append(envInitial, osEnv[i])
}
}
}
return envInitial
}
var envInitial []string
// createTestEnvironment creates a temporary testFolder and
// sets testConfig to testFolder/rclone.config.
func createTestEnvironment(t *testing.T) {
//Set temporary folder for config and test data
tempFolder, err := ioutil.TempDir("", "rclone_cmdtest_")
require.NoError(t, err)
testFolder = filepath.ToSlash(tempFolder)
// Set path to temporary config file
testConfig = testFolder + "/rclone.config"
}
var testFolder string
var testConfig string
// removeTestEnvironment removes the test environment created by createTestEnvironment
func removeTestEnvironment(t *testing.T) {
// Remove temporary folder with all contents
err := os.RemoveAll(testFolder)
require.NoError(t, err)
}
// createTestFile creates the file testFolder/name
func createTestFile(name string, t *testing.T) string {
err := ioutil.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666)
require.NoError(t, err)
return testFolder + "/" + name
}
// createTestFolder creates the folder testFolder/name
func createTestFolder(name string, t *testing.T) string {
err := os.Mkdir(testFolder+"/"+name, 0777)
require.NoError(t, err)
return testFolder + "/" + name
}
// createSimpleTestData creates simple test data in testFolder/subFolder
func createSimpleTestData(t *testing.T) string {
createTestFolder("testdata", t)
createTestFile("testdata/file1.txt", t)
createTestFile("testdata/file2.txt", t)
createTestFolder("testdata/folderA", t)
createTestFile("testdata/folderA/fileA1.txt", t)
createTestFile("testdata/folderA/fileA2.txt", t)
createTestFolder("testdata/folderA/folderAA", t)
createTestFile("testdata/folderA/folderAA/fileAA1.txt", t)
createTestFile("testdata/folderA/folderAA/fileAA2.txt", t)
createTestFolder("testdata/folderB", t)
createTestFile("testdata/folderB/fileB1.txt", t)
createTestFile("testdata/folderB/fileB2.txt", t)
return testFolder + "/testdata"
}
// removeSimpleTestData removes the test data created by createSimpleTestData
func removeSimpleTestData(t *testing.T) {
err := os.RemoveAll(testFolder + "/testdata")
require.NoError(t, err)
}
// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
func TestCmdTest(t *testing.T) {
createTestEnvironment(t)
defer removeTestEnvironment(t)
// Test simple call and output from rclone
out, err := rclone("version")
t.Logf("rclone version\n" + out)
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone v")
assert.Contains(t, out, "version: ")
assert.NotContains(t, out, "Error:")
assert.NotContains(t, out, "--help")
assert.NotContains(t, out, " DEBUG : ")
assert.Regexp(t, "rclone\\s+v\\d+\\.\\d+", out) // rclone v_.__
}
// Test multiple arguments and DEBUG output
out, err = rclone("version", "-vv")
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone v")
assert.Contains(t, out, " DEBUG : ")
}
// Test error and error output
out, err = rclone("version", "--provoke-an-error")
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "exit status 1")
assert.Contains(t, out, "Error: unknown flag")
}
// Test effect of environment variable
env := "RCLONE_LOG_LEVEL=DEBUG"
out, err = rcloneEnv(env, "version")
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone v")
assert.Contains(t, out, " DEBUG : ")
}
// Test effect of multiple environment variables, including one with ,
env = "RCLONE_LOG_LEVEL=DEBUG;RCLONE_LOG_FORMAT=date,shortfile;RCLONE_STATS=173ms"
out, err = rcloneEnv(env, "version")
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone v")
assert.Contains(t, out, " DEBUG : ")
assert.Regexp(t, "[^\\s]+\\.go:\\d+:", out) // ___.go:__:
assert.Contains(t, out, "173ms")
}
// Test setup of config file
out, err = rclone("config", "create", "myLocal", "local")
if assert.NoError(t, err) {
assert.Contains(t, out, "[myLocal]")
assert.Contains(t, out, "type = local")
}
// Test creation of simple test data
createSimpleTestData(t)
defer removeSimpleTestData(t)
// Test access to config file and simple test data
out, err = rclone("lsl", "myLocal:"+testFolder)
t.Logf("rclone lsl myLocal:testFolder\n" + out)
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone.config")
assert.Contains(t, out, "testdata/folderA/fileA1.txt")
}
}

276
cmdtest/environment_test.go Normal file
View File

@ -0,0 +1,276 @@
// environment_test tests the use and precedence of environment variables
//
// The tests rely on functions defined in cmdtest_test.go
package cmdtest
import (
"os"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
func TestEnvironmentVariables(t *testing.T) {
createTestEnvironment(t)
defer removeTestEnvironment(t)
testdataPath := createSimpleTestData(t)
defer removeSimpleTestData(t)
// Non backend flags
// =================
// First verify default behaviour of the implicit max_depth=-1
env := ""
out, err := rcloneEnv(env, "lsl", testFolder)
//t.Logf("\n" + out)
if assert.NoError(t, err) {
assert.Contains(t, out, "rclone.config") // depth 1
assert.Contains(t, out, "file1.txt") // depth 2
assert.Contains(t, out, "fileA1.txt") // depth 3
assert.Contains(t, out, "fileAA1.txt") // depth 4
}
// Test of flag.Value
env = "RCLONE_MAX_DEPTH=2"
out, err = rcloneEnv(env, "lsl", testFolder)
if assert.NoError(t, err) {
assert.Contains(t, out, "file1.txt") // depth 2
assert.NotContains(t, out, "fileA1.txt") // depth 3
}
// Test of flag.Changed (tests #5341 Issue1)
env = "RCLONE_LOG_LEVEL=DEBUG"
out, err = rcloneEnv(env, "version", "--quiet")
if assert.Error(t, err) {
assert.Contains(t, out, " DEBUG : ")
assert.Contains(t, out, "Can't set -q and --log-level")
assert.Contains(t, "exit status 1", err.Error())
}
// Test of flag.DefValue
env = "RCLONE_STATS=173ms"
out, err = rcloneEnv(env, "help", "flags")
if assert.NoError(t, err) {
assert.Contains(t, out, "(default 173ms)")
}
// Test of command line flags overriding environment flags
env = "RCLONE_MAX_DEPTH=2"
out, err = rcloneEnv(env, "lsl", testFolder, "--max-depth", "3")
if assert.NoError(t, err) {
assert.Contains(t, out, "fileA1.txt") // depth 3
assert.NotContains(t, out, "fileAA1.txt") // depth 4
}
// Test of debug logging while initialising flags from environment (tests #5241 Enhance1)
env = "RCLONE_STATS=173ms"
out, err = rcloneEnv(env, "version", "-vv")
if assert.NoError(t, err) {
assert.Contains(t, out, " DEBUG : ")
assert.Contains(t, out, "--stats")
assert.Contains(t, out, "173ms")
assert.Contains(t, out, "RCLONE_STATS=")
}
// Backend flags and option precedence
// ===================================
// Test approach:
// Verify no symlink warning when skip_links=true one the level with highest precedence
// and skip_links=false on all levels with lower precedence
//
// Reference: https://rclone.org/docs/#precedence
// Create a symlink in test data
err = os.Symlink(testdataPath+"/folderA", testdataPath+"/symlinkA")
if runtime.GOOS == "windows" {
errNote := "The policy settings on Windows often prohibit the creation of symlinks due to security issues.\n"
errNote += "You can safely ignore this test, if your change didn't affect environment variables."
require.NoError(t, err, errNote)
} else {
require.NoError(t, err)
}
// Create a local remote with explicit skip_links=false
out, err = rclone("config", "create", "myLocal", "local", "skip_links", "false")
if assert.NoError(t, err) {
assert.Contains(t, out, "[myLocal]")
assert.Contains(t, out, "type = local")
assert.Contains(t, out, "skip_links = false")
}
// Verify symlink warning when skip_links=false on all levels
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
out, err = rcloneEnv(env, "lsd", "myLocal,skip_links=false:"+testdataPath, "--skip-links=false")
//t.Logf("\n" + out)
if assert.NoError(t, err) {
assert.Contains(t, out, "NOTICE: symlinkA:")
assert.Contains(t, out, "folderA")
}
// Test precedence of connection strings
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
out, err = rcloneEnv(env, "lsd", "myLocal,skip_links:"+testdataPath, "--skip-links=false")
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test precedence of command line flags
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath, "--skip-links")
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test precedence of remote specific environment variables (tests #5341 Issue2)
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test precedence of backend specific environment variables (tests #5341 Issue3)
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test precedence of backend generic environment variables
env = "RCLONE_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Recreate the test remote with explicit skip_links=true
out, err = rclone("config", "create", "myLocal", "local", "skip_links", "true")
if assert.NoError(t, err) {
assert.Contains(t, out, "[myLocal]")
assert.Contains(t, out, "type = local")
assert.Contains(t, out, "skip_links = true")
}
// Test precedence of config file options
env = ""
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Recreate the test remote with rclone defaults, that is implicit skip_links=false
out, err = rclone("config", "create", "myLocal", "local")
if assert.NoError(t, err) {
assert.Contains(t, out, "[myLocal]")
assert.Contains(t, out, "type = local")
assert.NotContains(t, out, "skip_links")
}
// Verify the rclone default value (implicit skip_links=false)
env = ""
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.Contains(t, out, "NOTICE: symlinkA:")
assert.Contains(t, out, "folderA")
}
// Display of backend defaults (tests #4659)
//------------------------------------------
env = "RCLONE_DRIVE_CHUNK_SIZE=111M"
out, err = rcloneEnv(env, "help", "flags")
if assert.NoError(t, err) {
assert.Regexp(t, "--drive-chunk-size[^\\(]+\\(default 111M\\)", out)
}
// Options on referencing remotes (alias, crypt, etc.)
//----------------------------------------------------
// Create alias remote on myLocal having implicit skip_links=false
out, err = rclone("config", "create", "myAlias", "alias", "remote", "myLocal:"+testdataPath)
if assert.NoError(t, err) {
assert.Contains(t, out, "[myAlias]")
assert.Contains(t, out, "type = alias")
assert.Contains(t, out, "remote = myLocal:")
}
// Verify symlink warnings on the alias
env = ""
out, err = rcloneEnv(env, "lsd", "myAlias:")
if assert.NoError(t, err) {
assert.Contains(t, out, "NOTICE: symlinkA:")
assert.Contains(t, out, "folderA")
}
// Test backend generic flags
// having effect on the underlying local remote
env = "RCLONE_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myAlias:")
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test backend specific flags
// having effect on the underlying local remote
env = "RCLONE_LOCAL_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myAlias:")
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test remote specific flags
// having no effect unless supported by the immediate remote (alias)
env = "RCLONE_CONFIG_MYALIAS_SKIP_LINKS=true"
out, err = rcloneEnv(env, "lsd", "myAlias:")
if assert.NoError(t, err) {
assert.Contains(t, out, "NOTICE: symlinkA:")
assert.Contains(t, out, "folderA")
}
env = "RCLONE_CONFIG_MYALIAS_REMOTE=" + "myLocal:" + testdataPath + "/folderA"
out, err = rcloneEnv(env, "lsl", "myAlias:")
if assert.NoError(t, err) {
assert.Contains(t, out, "fileA1.txt")
assert.NotContains(t, out, "fileB1.txt")
}
// Test command line flags
// having effect on the underlying local remote
env = ""
out, err = rcloneEnv(env, "lsd", "myAlias:", "--skip-links")
if assert.NoError(t, err) {
assert.NotContains(t, out, "symlinkA")
assert.Contains(t, out, "folderA")
}
// Test connection specific flags
// having no effect unless supported by the immediate remote (alias)
env = ""
out, err = rcloneEnv(env, "lsd", "myAlias,skip_links:")
if assert.NoError(t, err) {
assert.Contains(t, out, "NOTICE: symlinkA:")
assert.Contains(t, out, "folderA")
}
env = ""
out, err = rcloneEnv(env, "lsl", "myAlias,remote='myLocal:"+testdataPath+"/folderA':", "-vv")
if assert.NoError(t, err) {
assert.Contains(t, out, "fileA1.txt")
assert.NotContains(t, out, "fileB1.txt")
}
}

View File

@ -239,6 +239,13 @@ However using the connection string syntax, this does work.
rclone copy "gdrive,shared_with_me:shared-file.txt" gdrive:
Note that the connection string only affects the options of the immediate
backend. If for example gdriveCrypt is a crypt based on gdrive, then the
following command **will not work** as intended, because
`shared_with_me` is ignored by the crypt backend:
rclone copy "gdriveCrypt,shared_with_me:shared-file.txt" gdriveCrypt:
The connection strings have the following syntax
remote,parameter=value,parameter2=value2:path/to/dir
@ -2157,6 +2164,11 @@ mys3:
Note that if you want to create a remote using environment variables
you must create the `..._TYPE` variable as above.
Note that you can only set the options of the immediate backend,
so RCLONE_CONFIG_MYS3CRYPT_ACCESS_KEY_ID has no effect, if myS3Crypt is
a crypt remote based on an S3 remote. However RCLONE_S3_ACCESS_KEY_ID will
set the access key of all remotes using S3, including myS3Crypt.
Note also that now rclone has [connection strings](#connection-strings),
it is probably easier to use those instead which makes the above example