config: fixed issues with flags/options set by environment vars.

Some environment variables didn’t behave like their corresponding
command line flags. The affected flags were --stats, --log-level,
--separator, --multi-tread-streams, --rc-addr, --rc-user and --rc-pass.
Example:

    RCLONE_STATS='10s'
    rclone check remote: remote: --progress
    # Expected: rclone check remote: remote: --progress –-stats=10s
    # Actual: rclone check remote: remote: --progress

Remote specific options set by environment variables was overruled by
less specific backend options set by environment variables. Example:

    RCLONE_DRIVE_USE_TRASH='false'
    RCLONE_CONFIG_MYDRIVE_USE_TRASH='true'
    rclone deletefile myDrive:my-test-file
    # Expected: my-test-file is recoverable in the trash folder
    # Actual: my-test-file is permanently deleted (not recoverable)

Backend specific options set by environment variables was overruled by
general backend options set by environment variables. Example:

    RCLONE_SKIP_LINKS='true'
    RCLONE_LOCAL_SKIP_LINKS='false'
    rclone lsd local:
    # Expected result: Warnings when symlinks are skipped
    # Actual result: No warnings when symlinks are skipped
    # That is RCLONE_SKIP_LINKS takes precedence

The above issues have been fixed.

The debug logging (-vv) has been enhanced to show when flags are set by
environment variables.

The documentation has been enhanced with details on the precedence of
configuration options.

See pull request #5341 for more information.

cmd/cmd.go

@@ -539,7 +539,8 @@ func AddBackendFlags() {
 				if opt.IsPassword {
 					help += " (obscured)"
 				}
-				flag := flags.VarPF(pflag.CommandLine, opt, name, opt.ShortOpt, help)
+				flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
+				flags.SetDefaultFromEnv(pflag.CommandLine, name)
 				if _, isBool := opt.Default.(bool); isBool {
 					flag.NoOptDefVal = "true"
 				}

docs/content/docs.md

@@ -2129,6 +2129,8 @@ Or to always use the trash in drive `--drive-use-trash`, set
 The same parser is used for the options and the environment variables
 so they take exactly the same form.
 
+The options set by environment variables can be seen with the `-vv` flag, e.g. `rclone version -vv`.
+
 ### Config file ###
 
 You can set defaults for values in the config file on an individual
@@ -2165,16 +2167,20 @@ it is probably easier to use those instead which makes the above example
 The various different methods of backend configuration are read in
 this order and the first one with a value is used.
 
-- Flag values as supplied on the command line, e.g. `--drive-use-trash`.
+- Parameters in connection strings, e.g. `myRemote,skip_links:`
-- Remote specific environment vars, e.g. `RCLONE_CONFIG_MYREMOTE_USE_TRASH` (see above).
+- Flag values as supplied on the command line, e.g. `--skip-links`
-- Backend specific environment vars, e.g. `RCLONE_DRIVE_USE_TRASH`.
+- Remote specific environment vars, e.g. `RCLONE_CONFIG_MYREMOTE_SKIP_LINKS` (see above).
-- Config file, e.g. `use_trash = false`.
+- Backend specific environment vars, e.g. `RCLONE_LOCAL_SKIP_LINKS`.
-- Default values, e.g. `true` - these can't be changed.
+- Backend generic environment vars, e.g. `RCLONE_SKIP_LINKS`.
+- Config file, e.g. `skip_links = true`.
+- Default values, e.g. `false` - these can't be changed.
 
-So if both `--drive-use-trash` is supplied on the config line and an
+So if both `--skip-links` is supplied on the command line and an
-environment variable `RCLONE_DRIVE_USE_TRASH` is set, the command line
+environment variable `RCLONE_LOCAL_SKIP_LINKS` is set, the command line
 flag will take preference.
 
+The backend configurations set by environment variables can be seen with the `-vv` flag, e.g. `rclone about myRemote: -vv`.
+
 For non backend configuration the order is as follows:
 
 - Flag values as supplied on the command line, e.g. `--stats 5s`.
@@ -2189,3 +2195,5 @@ For non backend configuration the order is as follows:
     - The environment values may be either a complete URL or a "host[:port]" for, in which case the "http" scheme is assumed.
 - `USER` and `LOGNAME` values are used as fallbacks for current username. The primary method for looking up username is OS-specific: Windows API on Windows, real user ID in /etc/passwd on Unix systems. In the documentation the current username is simply referred to as `$USER`.
 - `RCLONE_CONFIG_DIR` - rclone **sets** this variable for use in config files and sub processes to point to the directory holding the config file.
+
+The options set by environment variables can be seen with the `-vv` and `--log-level=DEBUG` flags, e.g. `rclone version -vv`.

fs/config.go

@@ -3,6 +3,7 @@ package fs
 import (
 	"context"
 	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -169,6 +170,20 @@ func NewConfig() *ConfigInfo {
 	c.FsCacheExpireDuration = 300 * time.Second
 	c.FsCacheExpireInterval = 60 * time.Second
 
+	// Perform a simple check for debug flags to enable debug logging during the flag initialization
+	for argIndex, arg := range os.Args {
+		if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {
+			c.LogLevel = LogLevelDebug
+		}
+		if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {
+			c.LogLevel = LogLevelDebug
+		}
+	}
+	envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")
+	if found && envValue == "DEBUG" {
+		c.LogLevel = LogLevelDebug
+	}
+
 	return c
 }
 

fs/config/flags/flags.go

@@ -11,194 +11,206 @@ import (
 	"github.com/spf13/pflag"
 )
 
-// setDefaultFromEnv constructs a name from the flag passed in and
+// setValueFromEnv constructs a name from the flag passed in and
-// sets the default from the environment if possible.
+// sets the value and default from the environment if possible
-func setDefaultFromEnv(flags *pflag.FlagSet, name string) {
+// the value may be overridden when the command line is parsed
-	key := fs.OptionToEnv(name)
+//
-	newValue, found := os.LookupEnv(key)
+// Used to create non-backend flags like --stats
+func setValueFromEnv(flags *pflag.FlagSet, name string) {
+	envKey := fs.OptionToEnv(name)
+	envValue, found := os.LookupEnv(envKey)
 	if found {
 		flag := flags.Lookup(name)
 		if flag == nil {
 			log.Fatalf("Couldn't find flag --%q", name)
 		}
-		err := flag.Value.Set(newValue)
+		err := flags.Set(name, envValue)
 		if err != nil {
-			log.Fatalf("Invalid value for environment variable %q when setting default for --%s: %v", key, name, err)
+			log.Fatalf("Invalid value when setting --%s from environment variable %s=%q: %v", name, envKey, envValue, err)
 		}
-		fs.Debugf(nil, "Set default for --%q from %q to %q (%v)", name, key, newValue, flag.Value)
+		fs.Debugf(nil, "Setting --%s %q from environment variable %s=%q", name, flag.Value, envKey, envValue)
-		flag.DefValue = newValue
+		flag.DefValue = envValue
 	}
 }
 
-// StringP defines a flag which can be overridden by an environment variable
+// SetDefaultFromEnv constructs a name from the flag passed in and
+// sets the default from the environment if possible
+//
+// Used to create backend flags like --skip-links
+func SetDefaultFromEnv(flags *pflag.FlagSet, name string) {
+	envKey := fs.OptionToEnv(name)
+	envValue, found := os.LookupEnv(envKey)
+	if found {
+		flag := flags.Lookup(name)
+		if flag == nil {
+			log.Fatalf("Couldn't find flag --%q", name)
+		}
+		fs.Debugf(nil, "Setting default for %s=%q from environment variable %s", name, envValue, envKey)
+		//err = tempValue.Set()
+		flag.DefValue = envValue
+	}
+}
+
+// StringP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.StringP
 func StringP(name, shorthand string, value string, usage string) (out *string) {
 	out = pflag.StringP(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// StringVarP defines a flag which can be overridden by an environment variable
+// StringVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.StringVarP
 func StringVarP(flags *pflag.FlagSet, p *string, name, shorthand string, value string, usage string) {
 	flags.StringVarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// BoolP defines a flag which can be overridden by an environment variable
+// BoolP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.BoolP
 func BoolP(name, shorthand string, value bool, usage string) (out *bool) {
 	out = pflag.BoolP(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// BoolVarP defines a flag which can be overridden by an environment variable
+// BoolVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.BoolVarP
 func BoolVarP(flags *pflag.FlagSet, p *bool, name, shorthand string, value bool, usage string) {
 	flags.BoolVarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// IntP defines a flag which can be overridden by an environment variable
+// IntP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.IntP
 func IntP(name, shorthand string, value int, usage string) (out *int) {
 	out = pflag.IntP(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// Int64P defines a flag which can be overridden by an environment variable
+// Int64P defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.IntP
 func Int64P(name, shorthand string, value int64, usage string) (out *int64) {
 	out = pflag.Int64P(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// Int64VarP defines a flag which can be overridden by an environment variable
+// Int64VarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.Int64VarP
 func Int64VarP(flags *pflag.FlagSet, p *int64, name, shorthand string, value int64, usage string) {
 	flags.Int64VarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// IntVarP defines a flag which can be overridden by an environment variable
+// IntVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.IntVarP
 func IntVarP(flags *pflag.FlagSet, p *int, name, shorthand string, value int, usage string) {
 	flags.IntVarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// Uint32VarP defines a flag which can be overridden by an environment variable
+// Uint32VarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.Uint32VarP
 func Uint32VarP(flags *pflag.FlagSet, p *uint32, name, shorthand string, value uint32, usage string) {
 	flags.Uint32VarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// Float64P defines a flag which can be overridden by an environment variable
+// Float64P defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.Float64P
 func Float64P(name, shorthand string, value float64, usage string) (out *float64) {
 	out = pflag.Float64P(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// Float64VarP defines a flag which can be overridden by an environment variable
+// Float64VarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.Float64VarP
 func Float64VarP(flags *pflag.FlagSet, p *float64, name, shorthand string, value float64, usage string) {
 	flags.Float64VarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// DurationP defines a flag which can be overridden by an environment variable
+// DurationP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.DurationP
 func DurationP(name, shorthand string, value time.Duration, usage string) (out *time.Duration) {
 	out = pflag.DurationP(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// DurationVarP defines a flag which can be overridden by an environment variable
+// DurationVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.DurationVarP
 func DurationVarP(flags *pflag.FlagSet, p *time.Duration, name, shorthand string, value time.Duration, usage string) {
 	flags.DurationVarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// VarP defines a flag which can be overridden by an environment variable
+// VarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.VarP
 func VarP(value pflag.Value, name, shorthand, usage string) {
 	pflag.VarP(value, name, shorthand, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 }
 
-// FVarP defines a flag which can be overridden by an environment variable
+// FVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.VarP
 func FVarP(flags *pflag.FlagSet, value pflag.Value, name, shorthand, usage string) {
 	flags.VarP(value, name, shorthand, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// VarPF defines a flag which can be overridden by an environment variable
+// StringArrayP defines a flag which can be set by an environment variable
-//
-// It is a thin wrapper around pflag.VarPF
-func VarPF(flags *pflag.FlagSet, value pflag.Value, name, shorthand, usage string) *pflag.Flag {
-	flag := flags.VarPF(value, name, shorthand, usage)
-	setDefaultFromEnv(flags, name)
-	return flag
-}
-
-// StringArrayP defines a flag which can be overridden by an environment variable
 //
 // It sets one value only - command line flags can be used to set more.
 //
 // It is a thin wrapper around pflag.StringArrayP
 func StringArrayP(name, shorthand string, value []string, usage string) (out *[]string) {
 	out = pflag.StringArrayP(name, shorthand, value, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// StringArrayVarP defines a flag which can be overridden by an environment variable
+// StringArrayVarP defines a flag which can be set by an environment variable
 //
 // It sets one value only - command line flags can be used to set more.
 //
 // It is a thin wrapper around pflag.StringArrayVarP
 func StringArrayVarP(flags *pflag.FlagSet, p *[]string, name, shorthand string, value []string, usage string) {
 	flags.StringArrayVarP(p, name, shorthand, value, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }
 
-// CountP defines a flag which can be overridden by an environment variable
+// CountP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.CountP
 func CountP(name, shorthand string, usage string) (out *int) {
 	out = pflag.CountP(name, shorthand, usage)
-	setDefaultFromEnv(pflag.CommandLine, name)
+	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
-// CountVarP defines a flag which can be overridden by an environment variable
+// CountVarP defines a flag which can be set by an environment variable
 //
 // It is a thin wrapper around pflag.CountVarP
 func CountVarP(flags *pflag.FlagSet, p *int, name, shorthand string, usage string) {
 	flags.CountVarP(p, name, shorthand, usage)
-	setDefaultFromEnv(flags, name)
+	setValueFromEnv(flags, name)
 }

fs/configmap.go

@@ -14,7 +14,12 @@ type configEnvVars string
 
 // Get a config item from the environment variables if possible
 func (configName configEnvVars) Get(key string) (value string, ok bool) {
-	return os.LookupEnv(ConfigToEnv(string(configName), key))
+	envKey := ConfigToEnv(string(configName), key)
+	value, ok = os.LookupEnv(envKey)
+	if ok {
+		Debugf(nil, "Setting %s=%q for %q from environment variable %s", key, value, configName, envKey)
+	}
+	return value, ok
 }
 
 // A configmap.Getter to read from the environment RCLONE_option_name
@@ -28,14 +33,19 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {
 	if opt == nil {
 		return "", false
 	}
-	// For options with NoPrefix set, check without prefix too
+	envKey := OptionToEnv(oev.fsInfo.Prefix + "-" + key)
-	if opt.NoPrefix {
+	value, ok = os.LookupEnv(envKey)
-		value, ok = os.LookupEnv(OptionToEnv(key))
+	if ok {
+		Debugf(nil, "Setting %s_%s=%q from environment variable %s", oev.fsInfo.Prefix, key, value, envKey)
+	} else if opt.NoPrefix {
+		// For options with NoPrefix set, check without prefix too
+		envKey := OptionToEnv(key)
+		value, ok = os.LookupEnv(envKey)
 		if ok {
-			return value, ok
+			Debugf(nil, "Setting %s=%q for %s from environment variable %s", key, value, oev.fsInfo.Prefix, envKey)
 		}
 	}
-	return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
+	return value, ok
 }
 
 // A configmap.Getter to read either the default value or the set

fs/rc/registry.go

@@ -7,8 +7,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-
-	"github.com/rclone/rclone/fs"
 )
 
 // Func defines a type for a remote control function
@@ -45,7 +43,7 @@ func (r *Registry) Add(call Call) {
 	defer r.mu.Unlock()
 	call.Path = strings.Trim(call.Path, "/")
 	call.Help = strings.TrimSpace(call.Help)
-	fs.Debugf(nil, "Adding path %q to remote control registry", call.Path)
+	// fs.Debugf(nil, "Adding path %q to remote control registry", call.Path) // disabled to make initialization less verbose
 	r.call[call.Path] = &call
 }