diff --git a/fs/config/configstruct/configstruct.go b/fs/config/configstruct/configstruct.go new file mode 100644 index 000000000..e2f57c4c0 --- /dev/null +++ b/fs/config/configstruct/configstruct.go @@ -0,0 +1,127 @@ +// Package configstruct parses unstructured maps into structures +package configstruct + +import ( + "fmt" + "reflect" + "regexp" + "strings" + + "github.com/ncw/rclone/fs/config/configmap" + "github.com/pkg/errors" +) + +var matchUpper = regexp.MustCompile("([A-Z]+)") + +// camelToSnake converts CamelCase to snake_case +func camelToSnake(in string) string { + out := matchUpper.ReplaceAllString(in, "_$1") + out = strings.ToLower(out) + out = strings.Trim(out, "_") + return out +} + +// StringToInterface turns in into an interface{} the same type as def +func StringToInterface(def interface{}, in string) (newValue interface{}, err error) { + typ := reflect.TypeOf(def) + switch typ.Kind() { + case reflect.String: + // Pass strings unmodified + return in, nil + } + // Otherwise parse with Sscanln + // + // This means any types we use here must implement fmt.Scanner + o := reflect.New(typ) + n, err := fmt.Sscanln(in, o.Interface()) + if err != nil { + return newValue, errors.Wrapf(err, "parsing %q as %T failed", in, def) + } + if n != 1 { + return newValue, errors.New("no items parsed") + } + return o.Elem().Interface(), nil +} + +// Item descripts a single entry in the options structure +type Item struct { + Name string // snake_case + Field string // CamelCase + Num int // number of the field in the struct + Value interface{} +} + +// Items parses the opt struct and returns a slice of Item objects. +// +// opt must be a pointer to a struct. The struct should have entirely +// public fields. +// +// The config_name is looked up in a struct tag called "config" or if +// not found is the field name converted from CamelCase to snake_case. +func Items(opt interface{}) (items []Item, err error) { + def := reflect.ValueOf(opt) + if def.Kind() != reflect.Ptr { + return nil, errors.New("argument must be a pointer") + } + def = def.Elem() // indirect the pointer + if def.Kind() != reflect.Struct { + return nil, errors.New("argument must be a pointer to a struct") + } + defType := def.Type() + for i := 0; i < def.NumField(); i++ { + field := defType.Field(i) + fieldName := field.Name + configName, ok := field.Tag.Lookup("config") + if !ok { + configName = camelToSnake(fieldName) + } + defaultItem := Item{ + Name: configName, + Field: fieldName, + Num: i, + Value: def.Field(i).Interface(), + } + items = append(items, defaultItem) + } + return items, nil +} + +// Set interprets the field names in defaults and looks up config +// values in the config passed in. Any values found in config will be +// set in the opt structure. +// +// opt must be a pointer to a struct. The struct should have entirely +// public fields. The field names are converted from CamelCase to +// snake_case and looked up in the config supplied or a +// `config:"field_name"` is looked up. +// +// If items are found then they are converted from string to native +// types and set in opt. +// +// All the field types in the struct must implement fmt.Scanner. +func Set(config configmap.Getter, opt interface{}) (err error) { + defaultItems, err := Items(opt) + if err != nil { + return err + } + defStruct := reflect.ValueOf(opt).Elem() + for _, defaultItem := range defaultItems { + newValue := defaultItem.Value + if configValue, ok := config.Get(defaultItem.Name); ok { + var newNewValue interface{} + newNewValue, err = StringToInterface(newValue, configValue) + if err != nil { + // Mask errors if setting an empty string as + // it isn't valid for all types. This makes + // empty string be the equivalent of unset. + if configValue != "" { + return errors.Wrapf(err, "couldn't parse config item %q = %q as %T", defaultItem.Name, configValue, defaultItem.Value) + } + } else { + newValue = newNewValue + } + } + defStruct.Field(defaultItem.Num).Set(reflect.ValueOf(newValue)) + } + return nil +} diff --git a/fs/config/configstruct/configstruct_test.go b/fs/config/configstruct/configstruct_test.go new file mode 100644 index 000000000..75e5fc29a --- /dev/null +++ b/fs/config/configstruct/configstruct_test.go @@ -0,0 +1,116 @@ +package configstruct_test + +import ( + "testing" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config/configstruct" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type conf struct { + A string + B string +} + +type conf2 struct { + PotatoPie string `config:"spud_pie"` + BeanStew bool + RaisinRoll int + SausageOnStick int64 + ForbiddenFruit uint + CookingTime fs.Duration + TotalWeight fs.SizeSuffix +} + +func TestItemsError(t *testing.T) { + _, err := configstruct.Items(nil) + assert.EqualError(t, err, "argument must be a pointer") + _, err = configstruct.Items(new(int)) + assert.EqualError(t, err, "argument must be a pointer to a struct") +} + +func TestItems(t *testing.T) { + in := &conf2{ + PotatoPie: "yum", + BeanStew: true, + RaisinRoll: 42, + SausageOnStick: 101, + ForbiddenFruit: 6, + CookingTime: fs.Duration(42 * time.Second), + TotalWeight: fs.SizeSuffix(17 << 20), + } + got, err := configstruct.Items(in) + require.NoError(t, err) + want := []configstruct.Item{ + {Name: "spud_pie", Field: "PotatoPie", Num: 0, Value: string("yum")}, + {Name: "bean_stew", Field: "BeanStew", Num: 1, Value: true}, + {Name: "raisin_roll", Field: "RaisinRoll", Num: 2, Value: int(42)}, + {Name: "sausage_on_stick", Field: "SausageOnStick", Num: 3, Value: int64(101)}, + {Name: "forbidden_fruit", Field: "ForbiddenFruit", Num: 4, Value: uint(6)}, + {Name: "cooking_time", Field: "CookingTime", Num: 5, Value: fs.Duration(42 * time.Second)}, + {Name: "total_weight", Field: "TotalWeight", Num: 6, Value: fs.SizeSuffix(17 << 20)}, + } + assert.Equal(t, want, got) +} + +func TestSetBasics(t *testing.T) { + c := &conf{A: "one", B: "two"} + err := configstruct.Set(configMap{}, c) + require.NoError(t, err) + assert.Equal(t, &conf{A: "one", B: "two"}, c) +} + +// a simple configmap.Getter for testing +type configMap map[string]string + +// Get the value +func (c configMap) Get(key string) (value string, ok bool) { + value, ok = c[key] + return value, ok +} + +func TestSetMore(t *testing.T) { + c := &conf{A: "one", B: "two"} + m := configMap{ + "a": "ONE", + } + err := configstruct.Set(m, c) + require.NoError(t, err) + assert.Equal(t, &conf{A: "ONE", B: "two"}, c) +} + +func TestSetFull(t *testing.T) { + in := &conf2{ + PotatoPie: "yum", + BeanStew: true, + RaisinRoll: 42, + SausageOnStick: 101, + ForbiddenFruit: 6, + CookingTime: fs.Duration(42 * time.Second), + TotalWeight: fs.SizeSuffix(17 << 20), + } + m := configMap{ + "spud_pie": "YUM", + "bean_stew": "FALSE", + "raisin_roll": "43 ", + "sausage_on_stick": " 102 ", + "forbidden_fruit": "0x7", + "cooking_time": "43s", + "total_weight": "18M", + } + want := &conf2{ + PotatoPie: "YUM", + BeanStew: false, + RaisinRoll: 43, + SausageOnStick: 102, + ForbiddenFruit: 7, + CookingTime: fs.Duration(43 * time.Second), + TotalWeight: fs.SizeSuffix(18 << 20), + } + err := configstruct.Set(m, in) + require.NoError(t, err) + assert.Equal(t, want, in) +} diff --git a/fs/config/configstruct/internal_test.go b/fs/config/configstruct/internal_test.go new file mode 100644 index 000000000..3918303e5 --- /dev/null +++ b/fs/config/configstruct/internal_test.go @@ -0,0 +1,60 @@ +package configstruct + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCamelToSnake(t *testing.T) { + for _, test := range []struct { + in string + want string + }{ + {"", ""}, + {"Type", "type"}, + {"AuthVersion", "auth_version"}, + {"AccessKeyID", "access_key_id"}, + } { + got := camelToSnake(test.in) + assert.Equal(t, test.want, got, test.in) + } +} + +func TestStringToInterface(t *testing.T) { + item := struct{ A int }{2} + for _, test := range []struct { + in string + def interface{} + want interface{} + err string + }{ + {"", string(""), "", ""}, + {" string ", string(""), " string ", ""}, + {"123", int(0), int(123), ""}, + {"0x123", int(0), int(0x123), ""}, + {" 0x123 ", int(0), int(0x123), ""}, + {"-123", int(0), int(-123), ""}, + {"0", false, false, ""}, + {"1", false, true, ""}, + {"FALSE", false, false, ""}, + {"true", false, true, ""}, + {"123", uint(0), uint(123), ""}, + {"123", int64(0), int64(123), ""}, + {"123x", int64(0), nil, "parsing \"123x\" as int64 failed: expected newline"}, + {"truth", false, nil, "parsing \"truth\" as bool failed: syntax error scanning boolean"}, + {"struct", item, nil, "parsing \"struct\" as struct { A int } failed: can't scan type: *struct { A int }"}, + } { + what := fmt.Sprintf("parse %q as %T", test.in, test.def) + got, err := StringToInterface(test.def, test.in) + if test.err == "" { + require.NoError(t, err, what) + assert.Equal(t, test.want, got, what) + } else { + assert.Nil(t, got) + assert.EqualError(t, err, test.err, what) + } + } +}