config: add configstruct parser to parse maps into config structures

fs/config/configstruct/configstruct.go

@@ -0,0 +1,127 @@
+// Package configstruct parses unstructured maps into structures
+package configstruct
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/pkg/errors"
+)
+
+var matchUpper = regexp.MustCompile("([A-Z]+)")
+
+// camelToSnake converts CamelCase to snake_case
+func camelToSnake(in string) string {
+	out := matchUpper.ReplaceAllString(in, "_$1")
+	out = strings.ToLower(out)
+	out = strings.Trim(out, "_")
+	return out
+}
+
+// StringToInterface turns in into an interface{} the same type as def
+func StringToInterface(def interface{}, in string) (newValue interface{}, err error) {
+	typ := reflect.TypeOf(def)
+	switch typ.Kind() {
+	case reflect.String:
+		// Pass strings unmodified
+		return in, nil
+	}
+	// Otherwise parse with Sscanln
+	//
+	// This means any types we use here must implement fmt.Scanner
+	o := reflect.New(typ)
+	n, err := fmt.Sscanln(in, o.Interface())
+	if err != nil {
+		return newValue, errors.Wrapf(err, "parsing %q as %T failed", in, def)
+	}
+	if n != 1 {
+		return newValue, errors.New("no items parsed")
+	}
+	return o.Elem().Interface(), nil
+}
+
+// Item descripts a single entry in the options structure
+type Item struct {
+	Name  string // snake_case
+	Field string // CamelCase
+	Num   int    // number of the field in the struct
+	Value interface{}
+}
+
+// Items parses the opt struct and returns a slice of Item objects.
+//
+// opt must be a pointer to a struct.  The struct should have entirely
+// public fields.
+//
+// The config_name is looked up in a struct tag called "config" or if
+// not found is the field name converted from CamelCase to snake_case.
+func Items(opt interface{}) (items []Item, err error) {
+	def := reflect.ValueOf(opt)
+	if def.Kind() != reflect.Ptr {
+		return nil, errors.New("argument must be a pointer")
+	}
+	def = def.Elem() // indirect the pointer
+	if def.Kind() != reflect.Struct {
+		return nil, errors.New("argument must be a pointer to a struct")
+	}
+	defType := def.Type()
+	for i := 0; i < def.NumField(); i++ {
+		field := defType.Field(i)
+		fieldName := field.Name
+		configName, ok := field.Tag.Lookup("config")
+		if !ok {
+			configName = camelToSnake(fieldName)
+		}
+		defaultItem := Item{
+			Name:  configName,
+			Field: fieldName,
+			Num:   i,
+			Value: def.Field(i).Interface(),
+		}
+		items = append(items, defaultItem)
+	}
+	return items, nil
+}
+
+// Set interprets the field names in defaults and looks up config
+// values in the config passed in.  Any values found in config will be
+// set in the opt structure.
+//
+// opt must be a pointer to a struct.  The struct should have entirely
+// public fields.  The field names are converted from CamelCase to
+// snake_case and looked up in the config supplied or a
+// `config:"field_name"` is looked up.
+//
+// If items are found then they are converted from string to native
+// types and set in opt.
+//
+// All the field types in the struct must implement fmt.Scanner.
+func Set(config configmap.Getter, opt interface{}) (err error) {
+	defaultItems, err := Items(opt)
+	if err != nil {
+		return err
+	}
+	defStruct := reflect.ValueOf(opt).Elem()
+	for _, defaultItem := range defaultItems {
+		newValue := defaultItem.Value
+		if configValue, ok := config.Get(defaultItem.Name); ok {
+			var newNewValue interface{}
+			newNewValue, err = StringToInterface(newValue, configValue)
+			if err != nil {
+				// Mask errors if setting an empty string as
+				// it isn't valid for all types.  This makes
+				// empty string be the equivalent of unset.
+				if configValue != "" {
+					return errors.Wrapf(err, "couldn't parse config item %q = %q as %T", defaultItem.Name, configValue, defaultItem.Value)
+				}
+			} else {
+				newValue = newNewValue
+			}
+		}
+		defStruct.Field(defaultItem.Num).Set(reflect.ValueOf(newValue))
+	}
+	return nil
+}

fs/config/configstruct/configstruct_test.go

@@ -0,0 +1,116 @@
+package configstruct_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type conf struct {
+	A string
+	B string
+}
+
+type conf2 struct {
+	PotatoPie      string `config:"spud_pie"`
+	BeanStew       bool
+	RaisinRoll     int
+	SausageOnStick int64
+	ForbiddenFruit uint
+	CookingTime    fs.Duration
+	TotalWeight    fs.SizeSuffix
+}
+
+func TestItemsError(t *testing.T) {
+	_, err := configstruct.Items(nil)
+	assert.EqualError(t, err, "argument must be a pointer")
+	_, err = configstruct.Items(new(int))
+	assert.EqualError(t, err, "argument must be a pointer to a struct")
+}
+
+func TestItems(t *testing.T) {
+	in := &conf2{
+		PotatoPie:      "yum",
+		BeanStew:       true,
+		RaisinRoll:     42,
+		SausageOnStick: 101,
+		ForbiddenFruit: 6,
+		CookingTime:    fs.Duration(42 * time.Second),
+		TotalWeight:    fs.SizeSuffix(17 << 20),
+	}
+	got, err := configstruct.Items(in)
+	require.NoError(t, err)
+	want := []configstruct.Item{
+		{Name: "spud_pie", Field: "PotatoPie", Num: 0, Value: string("yum")},
+		{Name: "bean_stew", Field: "BeanStew", Num: 1, Value: true},
+		{Name: "raisin_roll", Field: "RaisinRoll", Num: 2, Value: int(42)},
+		{Name: "sausage_on_stick", Field: "SausageOnStick", Num: 3, Value: int64(101)},
+		{Name: "forbidden_fruit", Field: "ForbiddenFruit", Num: 4, Value: uint(6)},
+		{Name: "cooking_time", Field: "CookingTime", Num: 5, Value: fs.Duration(42 * time.Second)},
+		{Name: "total_weight", Field: "TotalWeight", Num: 6, Value: fs.SizeSuffix(17 << 20)},
+	}
+	assert.Equal(t, want, got)
+}
+
+func TestSetBasics(t *testing.T) {
+	c := &conf{A: "one", B: "two"}
+	err := configstruct.Set(configMap{}, c)
+	require.NoError(t, err)
+	assert.Equal(t, &conf{A: "one", B: "two"}, c)
+}
+
+// a simple configmap.Getter for testing
+type configMap map[string]string
+
+// Get the value
+func (c configMap) Get(key string) (value string, ok bool) {
+	value, ok = c[key]
+	return value, ok
+}
+
+func TestSetMore(t *testing.T) {
+	c := &conf{A: "one", B: "two"}
+	m := configMap{
+		"a": "ONE",
+	}
+	err := configstruct.Set(m, c)
+	require.NoError(t, err)
+	assert.Equal(t, &conf{A: "ONE", B: "two"}, c)
+}
+
+func TestSetFull(t *testing.T) {
+	in := &conf2{
+		PotatoPie:      "yum",
+		BeanStew:       true,
+		RaisinRoll:     42,
+		SausageOnStick: 101,
+		ForbiddenFruit: 6,
+		CookingTime:    fs.Duration(42 * time.Second),
+		TotalWeight:    fs.SizeSuffix(17 << 20),
+	}
+	m := configMap{
+		"spud_pie":         "YUM",
+		"bean_stew":        "FALSE",
+		"raisin_roll":      "43 ",
+		"sausage_on_stick": " 102 ",
+		"forbidden_fruit":  "0x7",
+		"cooking_time":     "43s",
+		"total_weight":     "18M",
+	}
+	want := &conf2{
+		PotatoPie:      "YUM",
+		BeanStew:       false,
+		RaisinRoll:     43,
+		SausageOnStick: 102,
+		ForbiddenFruit: 7,
+		CookingTime:    fs.Duration(43 * time.Second),
+		TotalWeight:    fs.SizeSuffix(18 << 20),
+	}
+	err := configstruct.Set(m, in)
+	require.NoError(t, err)
+	assert.Equal(t, want, in)
+}

fs/config/configstruct/internal_test.go

@@ -0,0 +1,60 @@
+package configstruct
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCamelToSnake(t *testing.T) {
+	for _, test := range []struct {
+		in   string
+		want string
+	}{
+		{"", ""},
+		{"Type", "type"},
+		{"AuthVersion", "auth_version"},
+		{"AccessKeyID", "access_key_id"},
+	} {
+		got := camelToSnake(test.in)
+		assert.Equal(t, test.want, got, test.in)
+	}
+}
+
+func TestStringToInterface(t *testing.T) {
+	item := struct{ A int }{2}
+	for _, test := range []struct {
+		in   string
+		def  interface{}
+		want interface{}
+		err  string
+	}{
+		{"", string(""), "", ""},
+		{"   string   ", string(""), "   string   ", ""},
+		{"123", int(0), int(123), ""},
+		{"0x123", int(0), int(0x123), ""},
+		{"   0x123   ", int(0), int(0x123), ""},
+		{"-123", int(0), int(-123), ""},
+		{"0", false, false, ""},
+		{"1", false, true, ""},
+		{"FALSE", false, false, ""},
+		{"true", false, true, ""},
+		{"123", uint(0), uint(123), ""},
+		{"123", int64(0), int64(123), ""},
+		{"123x", int64(0), nil, "parsing \"123x\" as int64 failed: expected newline"},
+		{"truth", false, nil, "parsing \"truth\" as bool failed: syntax error scanning boolean"},
+		{"struct", item, nil, "parsing \"struct\" as struct { A int } failed: can't scan type: *struct { A int }"},
+	} {
+		what := fmt.Sprintf("parse %q as %T", test.in, test.def)
+		got, err := StringToInterface(test.def, test.in)
+		if test.err == "" {
+			require.NoError(t, err, what)
+			assert.Equal(t, test.want, got, what)
+		} else {
+			assert.Nil(t, got)
+			assert.EqualError(t, err, test.err, what)
+		}
+	}
+}