From c4451bc43a32a5117c0cbe5a96140c65ec6cf507 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 15:46:07 +0100 Subject: [PATCH] fs: add --metadata-set flag to specify metadata for uploads --- docs/content/docs.md | 8 ++- fs/config.go | 1 + fs/config/configflags/configflags.go | 16 ++++++ fs/metadata.go | 36 +++++++++++++ fs/metadata_test.go | 79 ++++++++++++++++++++++++++++ fs/open_options.go | 18 +++++++ fs/open_options_test.go | 10 ++++ fs/operations/operations.go | 6 +++ 8 files changed, 173 insertions(+), 1 deletion(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 7cfffe2b4..a2bc38b07 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not Windows). Note that arbitrary metadata may be added to objects using the -`--upload-metadata key=value` flag when the object is first uploaded. +`--metadata-set key=value` flag when the object is first uploaded. This flag can be repeated as many times as necessary. ### Types of metadata @@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source to the destination. For local backends this is ownership, permissions, xattr etc. See the [#metadata](metadata section) for more info. +### --metadata-set key=value + +Add metadata `key` = `value` when uploading. This can be repeated as +many times as required. See the [#metadata](metadata section) for more +info. + ### --cutoff-mode=hard|soft|cautious ### This modifies the behavior of `--max-transfer` diff --git a/fs/config.go b/fs/config.go index e96be37d0..dceb063bb 100644 --- a/fs/config.go +++ b/fs/config.go @@ -124,6 +124,7 @@ type ConfigInfo struct { UploadHeaders []*HTTPOption DownloadHeaders []*HTTPOption Headers []*HTTPOption + MetadataSet Metadata // extra metadata to write when uploading RefreshTimes bool NoConsole bool TrafficClass uint8 diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 8e3e8d558..de9d08073 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -37,6 +37,7 @@ var ( uploadHeaders []string downloadHeaders []string headers []string + metadataSet []string ) // AddFlags adds the non filing system specific flags to the command @@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions") flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions") flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions") + flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading") flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files") flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)") flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21") @@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) { if len(headers) != 0 { ci.Headers = ParseHeaders(headers) } + if len(headers) != 0 { + ci.Headers = ParseHeaders(headers) + } + if len(metadataSet) != 0 { + ci.MetadataSet = make(fs.Metadata, len(metadataSet)) + for _, kv := range metadataSet { + equal := strings.IndexRune(kv, '=') + if equal < 0 { + log.Fatalf("Failed to parse '%s' as metadata key=value.", kv) + } + ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:] + } + fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet) + } if len(dscp) != 0 { if value, ok := parseDSCP(dscp); ok { ci.TrafficClass = value << 2 diff --git a/fs/metadata.go b/fs/metadata.go index 3df5a29f5..50d0340ed 100644 --- a/fs/metadata.go +++ b/fs/metadata.go @@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) { (*m)[k] = v } +// Merge other into m +// +// If m is nil, then it will get made +func (m *Metadata) Merge(other Metadata) { + for k, v := range other { + if *m == nil { + *m = make(Metadata, len(other)) + } + (*m)[k] = v + } +} + +// MergeOptions gets any Metadata from the options passed in and +// stores it in m (which may be nil). +// +// If there is no m then metadata will be nil +func (m *Metadata) MergeOptions(options []OpenOption) { + for _, opt := range options { + if metadataOption, ok := opt.(MetadataOption); ok { + m.Merge(Metadata(metadataOption)) + } + } +} + // GetMetadata from an ObjectInfo // // If the object has no metadata then metadata will be nil @@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro } return do.Metadata(ctx) } + +// GetMetadataOptions from an ObjectInfo and merge it with any in options +// +// If the object has no metadata then metadata will be nil +func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) { + metadata, err = GetMetadata(ctx, o) + if err != nil { + return nil, err + } + metadata.MergeOptions(options) + return metadata, nil +} diff --git a/fs/metadata_test.go b/fs/metadata_test.go index bc72c3964..ba982cbc1 100644 --- a/fs/metadata_test.go +++ b/fs/metadata_test.go @@ -1,6 +1,7 @@ package fs import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) { m.Set("key", "value2") assert.Equal(t, "value2", m["key"]) } + +func TestMetadataMerge(t *testing.T) { + for _, test := range []struct { + in Metadata + merge Metadata + want Metadata + }{ + { + in: Metadata{}, + merge: Metadata{}, + want: Metadata{}, + }, { + in: nil, + merge: nil, + want: nil, + }, { + in: nil, + merge: Metadata{}, + want: nil, + }, { + in: nil, + merge: Metadata{"a": "1", "b": "2"}, + want: Metadata{"a": "1", "b": "2"}, + }, { + in: Metadata{"a": "1", "b": "2"}, + merge: nil, + want: Metadata{"a": "1", "b": "2"}, + }, { + in: Metadata{"a": "1", "b": "2"}, + merge: Metadata{"b": "B", "c": "3"}, + want: Metadata{"a": "1", "b": "B", "c": "3"}, + }, + } { + what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge) + test.in.Merge(test.merge) + assert.Equal(t, test.want, test.in, what) + } +} + +func TestMetadataMergeOptions(t *testing.T) { + for _, test := range []struct { + in Metadata + opts []OpenOption + want Metadata + }{ + { + opts: []OpenOption{}, + want: nil, + }, { + opts: []OpenOption{&HTTPOption{}}, + want: nil, + }, { + opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}}, + want: Metadata{"a": "1", "b": "2"}, + }, { + opts: []OpenOption{ + &HTTPOption{}, + MetadataOption{"a": "1", "b": "2"}, + MetadataOption{"b": "B", "c": "3"}, + &HTTPOption{}, + }, + want: Metadata{"a": "1", "b": "B", "c": "3"}, + }, { + in: Metadata{"a": "first", "z": "OK"}, + opts: []OpenOption{ + &HTTPOption{}, + MetadataOption{"a": "1", "b": "2"}, + MetadataOption{"b": "B", "c": "3"}, + &HTTPOption{}, + }, + want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"}, + }, + } { + what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts) + test.in.MergeOptions(test.opts) + assert.Equal(t, test.want, test.in, what) + } +} diff --git a/fs/open_options.go b/fs/open_options.go index e357a9a69..57daa132a 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool { return false } +// MetadataOption defines an Option which does nothing +type MetadataOption Metadata + +// Header formats the option as an http header +func (o MetadataOption) Header() (key string, value string) { + return "", "" +} + +// String formats the option into human-readable form +func (o MetadataOption) String() string { + return fmt.Sprintf("MetadataOption(%v)", Metadata(o)) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o MetadataOption) Mandatory() bool { + return false +} + // OpenOptionAddHeaders adds each header found in options to the // headers map provided the key was non empty. func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) { diff --git a/fs/open_options_test.go b/fs/open_options_test.go index 829f45c19..37463adef 100644 --- a/fs/open_options_test.go +++ b/fs/open_options_test.go @@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) { assert.Equal(t, false, opt.Mandatory()) } +func TestMetadataOption(t *testing.T) { + opt := MetadataOption{"onion": "ice cream"} + var _ OpenOption = opt // check interface + assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String()) + key, value := opt.Header() + assert.Equal(t, "", key) + assert.Equal(t, "", value) + assert.Equal(t, false, opt.Mandatory()) +} + func TestFixRangeOptions(t *testing.T) { for _, test := range []struct { name string diff --git a/fs/operations/operations.go b/fs/operations/operations.go index aa69a0cbf..a41a0a009 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj for _, option := range ci.UploadHeaders { options = append(options, option) } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } if doUpdate { actionTaken = "Copied (replaced existing)" err = dst.Update(ctx, in, wrappedSrc, options...) @@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, for _, option := range ci.UploadHeaders { options = append(options, option) } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } compare := func(dst fs.Object) error { var sums map[hash.Type]string