diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 900bb57e2..6d5a76d3a 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -48,6 +48,7 @@ const ( maxChunkSize = 100 * 1024 * 1024 defaultUploadCutoff = 256 * 1024 * 1024 maxUploadCutoff = 256 * 1024 * 1024 + defaultAccessTier = azblob.AccessTierNone ) // Register with Fs @@ -79,6 +80,11 @@ func init() { Help: "Upload chunk size. Must fit in memory.", Default: fs.SizeSuffix(defaultChunkSize), Advanced: true, + }, { + Name: "access_tier", + Help: "Access tier of blob, supports hot, cool and archive tiers.\nArchived blobs can be restored by setting access tier to hot or cool." + + " Leave blank if you intend to use default access tier, which is set at account level", + Advanced: true, }}, }) } @@ -91,6 +97,7 @@ type Options struct { SASURL string `config:"sas_url"` UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` ChunkSize fs.SizeSuffix `config:"chunk_size"` + AccessTier string `config:"access_tier"` } // Fs represents a remote azure server @@ -212,6 +219,19 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { opt.Endpoint = storageDefaultBaseURL } + if opt.AccessTier == "" { + opt.AccessTier = string(defaultAccessTier) + } else { + switch opt.AccessTier { + case string(azblob.AccessTierHot): + case string(azblob.AccessTierCool): + case string(azblob.AccessTierArchive): + // valid cases + default: + return nil, errors.Errorf("azure: Supported access tiers are %s, %s and %s", string(azblob.AccessTierHot), string(azblob.AccessTierCool), azblob.AccessTierArchive) + } + } + var ( u *url.URL serviceURL azblob.ServiceURL @@ -946,6 +966,10 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) { // Offset and Count for range download var offset int64 var count int64 + if o.AccessTier() == azblob.AccessTierArchive { + return nil, errors.Errorf("Blob in archive tier, you need to set tier to hot or cool first") + } + for _, option := range options { switch x := option.(type) { case *fs.RangeOption: @@ -1199,8 +1223,29 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio if err != nil { return err } + // Refresh metadata on object o.clearMetaData() - return o.readMetaData() + err = o.readMetaData() + if err != nil { + return err + } + + // If tier is not changed or not specified, do not attempt to invoke `SetBlobTier` operation + if o.fs.opt.AccessTier == string(defaultAccessTier) || o.fs.opt.AccessTier == string(o.AccessTier()) { + return nil + } + + // Now, set blob tier based on configured access tier + desiredAccessTier := azblob.AccessTierType(o.fs.opt.AccessTier) + err = o.fs.pacer.Call(func() (bool, error) { + _, err := blob.SetTier(ctx, desiredAccessTier) + return o.fs.shouldRetry(err) + }) + + if err != nil { + return errors.Wrap(err, "Failed to set Blob Tier") + } + return nil } // Remove an object @@ -1220,6 +1265,11 @@ func (o *Object) MimeType() string { return o.mimeType } +// AccessTier of an object, default is of type none +func (o *Object) AccessTier() azblob.AccessTierType { + return o.accessTier +} + // Check the interfaces are satisfied var ( _ fs.Fs = &Fs{} diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md index c4da72e79..84b67b35e 100644 --- a/docs/content/azureblob.md +++ b/docs/content/azureblob.md @@ -184,6 +184,17 @@ Upload chunk size. Default 4MB. Note that this is stored in memory and there may be up to `--transfers` chunks stored at once in memory. This can be at most 100MB. +#### --azureblob-access-tier=Hot/Cool/Archive #### + +Azure storage supports blob tiering, you can configure tier in advanced +settings or supply flag while performing data transfer operations. +If there is no `access tier` specified, rclone doesn't apply any tier. +rclone performs `Set Tier` operation on blobs while uploading, if objects +are not modified, specifying `access tier` to new one will have no effect. +If blobs are in `archive tier` at remote, trying to perform data transfer +operations from remote will not be allowed. User should first restore by +tiering blob to `Hot` or `Cool`. + ### Limitations ### MD5 sums are only uploaded with chunked files if the source has an MD5