HDFS (Hadoop Distributed File System) implementation - #42

This includes an HDFS docker image to use with the integration tests. Co-authored-by: Ivan Andreev <ivandeex@gmail.com> Co-authored-by: Nick Craig-Wood <nick@craig-wood.com>
2024-11-21 22:50:16 +01:00 · 2020-09-28 20:29:44 +03:00 · 2020-09-28 20:29:44 +03:00 · 71edc75ca6
commit 71edc75ca6
parent 768e4c4735
26 changed files with 906 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -36,6 +36,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
  * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
  * Google Drive [:page_facing_up:](https://rclone.org/drive/)
  * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
+  * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
  * HTTP [:page_facing_up:](https://rclone.org/http/)
  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
  * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
--- a/backend/all/all.go
+++ b/backend/all/all.go
@ -18,6 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/ftp"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/googlephotos"
+	_ "github.com/rclone/rclone/backend/hdfs"
 	_ "github.com/rclone/rclone/backend/http"
 	_ "github.com/rclone/rclone/backend/hubic"
 	_ "github.com/rclone/rclone/backend/jottacloud"
--- a/backend/hdfs/fs.go
+++ b/backend/hdfs/fs.go
@ -0,0 +1,257 @@
+// +build !plan9
+
+package hdfs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"time"
+
+	"github.com/colinmarc/hdfs/v2"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Fs represents a HDFS server
+type Fs struct {
+	name     string
+	root     string
+	features *fs.Features   // optional features
+	opt      Options        // options for this backend
+	ci       *fs.ConfigInfo // global config
+	client   *hdfs.Client
+}
+
+// NewFs constructs an Fs from the path
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := hdfs.NewClient(hdfs.ClientOptions{
+		Addresses:           []string{opt.Namenode},
+		User:                opt.Username,
+		UseDatanodeHostname: false,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	f := &Fs{
+		name:   name,
+		root:   root,
+		opt:    *opt,
+		ci:     fs.GetConfig(ctx),
+		client: client,
+	}
+
+	f.features = (&fs.Features{
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f)
+
+	info, err := f.client.Stat(f.realpath(""))
+	if err == nil && !info.IsDir() {
+		f.root = path.Dir(f.root)
+		return f, fs.ErrorIsFile
+	}
+
+	return f, nil
+}
+
+// Name of this fs
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("hdfs://%s", f.opt.Namenode)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision return the precision of this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Hashes are not supported
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.None)
+}
+
+// NewObject finds file at remote or return fs.ErrorObjectNotFound
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	realpath := f.realpath(remote)
+	fs.Debugf(f, "new [%s]", realpath)
+
+	info, err := f.ensureFile(realpath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Object{
+		fs:      f,
+		remote:  remote,
+		size:    info.Size(),
+		modTime: info.ModTime(),
+	}, nil
+}
+
+// List the objects and directories in dir into entries.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	realpath := f.realpath(dir)
+	fs.Debugf(f, "list [%s]", realpath)
+
+	err = f.ensureDirectory(realpath)
+	if err != nil {
+		return nil, err
+	}
+
+	list, err := f.client.ReadDir(realpath)
+	if err != nil {
+		return nil, err
+	}
+	for _, x := range list {
+		stdName := f.opt.Enc.ToStandardName(x.Name())
+		remote := path.Join(dir, stdName)
+		if x.IsDir() {
+			entries = append(entries, fs.NewDir(remote, x.ModTime()))
+		} else {
+			entries = append(entries, &Object{
+				fs:      f,
+				remote:  remote,
+				size:    x.Size(),
+				modTime: x.ModTime()})
+		}
+	}
+	return entries, nil
+}
+
+// Put the object
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o := &Object{
+		fs:     f,
+		remote: src.Remote(),
+	}
+	err := o.Update(ctx, in, src, options...)
+	return o, err
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+// Mkdir makes a directory
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	fs.Debugf(f, "mkdir [%s]", f.realpath(dir))
+	return f.client.MkdirAll(f.realpath(dir), 0755)
+}
+
+// Rmdir deletes the directory
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	realpath := f.realpath(dir)
+	fs.Debugf(f, "rmdir [%s]", realpath)
+
+	err := f.ensureDirectory(realpath)
+	if err != nil {
+		return err
+	}
+
+	// do not remove empty directory
+	list, err := f.client.ReadDir(realpath)
+	if err != nil {
+		return err
+	}
+	if len(list) > 0 {
+		return fs.ErrorDirectoryNotEmpty
+	}
+
+	return f.client.Remove(realpath)
+}
+
+// Purge deletes all the files in the directory
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	realpath := f.realpath(dir)
+	fs.Debugf(f, "purge [%s]", realpath)
+
+	err := f.ensureDirectory(realpath)
+	if err != nil {
+		return err
+	}
+
+	return f.client.RemoveAll(realpath)
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	info, err := f.client.StatFs()
+	if err != nil {
+		return nil, err
+	}
+	return &fs.Usage{
+		Total: fs.NewUsageValue(int64(info.Capacity)),
+		Used:  fs.NewUsageValue(int64(info.Used)),
+		Free:  fs.NewUsageValue(int64(info.Remaining)),
+	}, nil
+}
+
+func (f *Fs) ensureDirectory(realpath string) error {
+	info, err := f.client.Stat(realpath)
+
+	if e, ok := err.(*os.PathError); ok && e.Err == os.ErrNotExist {
+		return fs.ErrorDirNotFound
+	}
+	if err != nil {
+		return err
+	}
+	if !info.IsDir() {
+		return fs.ErrorDirNotFound
+	}
+
+	return nil
+}
+
+func (f *Fs) ensureFile(realpath string) (os.FileInfo, error) {
+	info, err := f.client.Stat(realpath)
+
+	if e, ok := err.(*os.PathError); ok && e.Err == os.ErrNotExist {
+		return nil, fs.ErrorObjectNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	if info.IsDir() {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	return info, nil
+}
+
+func (f *Fs) realpath(dir string) string {
+	return f.opt.Enc.FromStandardPath(xPath(f.Root(), dir))
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = (*Fs)(nil)
+	_ fs.Purger      = (*Fs)(nil)
+	_ fs.PutStreamer = (*Fs)(nil)
+	_ fs.Abouter     = (*Fs)(nil)
+)
--- a/backend/hdfs/hdfs.go
+++ b/backend/hdfs/hdfs.go
@ -0,0 +1,58 @@
+// +build !plan9
+
+package hdfs
+
+import (
+	"path"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/lib/encoder"
+)
+
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "hdfs",
+		Description: "Hadoop distributed file system",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Name:     "namenode",
+			Help:     "hadoop name node and port",
+			Required: true,
+			Examples: []fs.OptionExample{{
+				Value: "namenode:8020",
+				Help:  "Connect to host namenode at port 8020",
+			}},
+		}, {
+			Name:     "username",
+			Help:     "hadoop user name",
+			Required: false,
+			Examples: []fs.OptionExample{{
+				Value: "root",
+				Help:  "Connect to hdfs as root",
+			}},
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default:  (encoder.Display | encoder.EncodeInvalidUtf8 | encoder.EncodeColon),
+		}},
+	}
+	fs.Register(fsi)
+}
+
+// Options for this backend
+type Options struct {
+	Namenode string               `config:"namenode"`
+	Username string               `config:"username"`
+	Enc      encoder.MultiEncoder `config:"encoding"`
+}
+
+// xPath make correct file path with leading '/'
+func xPath(root string, tail string) string {
+	if !strings.HasPrefix(root, "/") {
+		root = "/" + root
+	}
+	return path.Join(root, tail)
+}
--- a/backend/hdfs/hdfs_test.go
+++ b/backend/hdfs/hdfs_test.go
@ -0,0 +1,20 @@
+// Test HDFS filesystem interface
+
+// +build !plan9
+
+package hdfs_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/hdfs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestHdfs:",
+		NilObject:  (*hdfs.Object)(nil),
+	})
+}
--- a/backend/hdfs/hdfs_unsupported.go
+++ b/backend/hdfs/hdfs_unsupported.go
@ -0,0 +1,6 @@
+// Build for hdfs for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build plan9
+
+package hdfs
--- a/backend/hdfs/object.go
+++ b/backend/hdfs/object.go
@ -0,0 +1,177 @@
+// +build !plan9
+
+package hdfs
+
+import (
+	"context"
+	"io"
+	"path"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/readers"
+)
+
+// Object describes an HDFS file
+type Object struct {
+	fs      *Fs
+	remote  string
+	size    int64
+	modTime time.Time
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// ModTime returns the modification time of the object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	realpath := o.fs.realpath(o.Remote())
+	err := o.fs.client.Chtimes(realpath, modTime, modTime)
+	if err != nil {
+		return err
+	}
+	o.modTime = modTime
+	return nil
+}
+
+// Storable returns whether this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Hash is not supported
+func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	realpath := o.realpath()
+	fs.Debugf(o.fs, "open [%s]", realpath)
+	f, err := o.fs.client.Open(realpath)
+	if err != nil {
+		return nil, err
+	}
+
+	var offset, limit int64 = 0, -1
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			offset = x.Offset
+		case *fs.RangeOption:
+			offset, limit = x.Decode(o.Size())
+		}
+	}
+
+	_, err = f.Seek(offset, io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+
+	if limit != -1 {
+		in = readers.NewLimitedReadCloser(f, limit)
+	} else {
+		in = f
+	}
+
+	return in, err
+}
+
+// Update object
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	realpath := o.fs.realpath(src.Remote())
+	dirname := path.Dir(realpath)
+	fs.Debugf(o.fs, "update [%s]", realpath)
+
+	err := o.fs.client.MkdirAll(dirname, 755)
+	if err != nil {
+		return err
+	}
+
+	info, err := o.fs.client.Stat(realpath)
+	if err == nil {
+		err = o.fs.client.Remove(realpath)
+		if err != nil {
+			return err
+		}
+	}
+
+	out, err := o.fs.client.Create(realpath)
+	if err != nil {
+		return err
+	}
+
+	cleanup := func() {
+		rerr := o.fs.client.Remove(realpath)
+		if rerr != nil {
+			fs.Errorf(o.fs, "failed to remove [%v]: %v", realpath, rerr)
+		}
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		cleanup()
+		return err
+	}
+
+	err = out.Close()
+	if err != nil {
+		cleanup()
+		return err
+	}
+
+	info, err = o.fs.client.Stat(realpath)
+	if err != nil {
+		return err
+	}
+
+	err = o.SetModTime(ctx, src.ModTime(ctx))
+	if err != nil {
+		return err
+	}
+	o.size = info.Size()
+
+	return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	realpath := o.fs.realpath(o.remote)
+	fs.Debugf(o.fs, "remove [%s]", realpath)
+	return o.fs.client.Remove(realpath)
+}
+
+func (o *Object) realpath() string {
+	return o.fs.opt.Enc.FromStandardPath(xPath(o.Fs().Root(), o.remote))
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Object = (*Object)(nil)
+)
--- a/bin/make_manual.py
+++ b/bin/make_manual.py
@ -42,6 +42,7 @@ docs = [
    "googlecloudstorage.md",
    "drive.md",
    "googlephotos.md",
+    "hdfs.md",
    "http.md",
    "hubic.md",
    "jottacloud.md",
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@ -120,6 +120,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
 {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}
 {{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}}
+{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
 {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
 {{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
 {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@ -36,6 +36,7 @@ See the following for detailed instructions for
  * [Google Cloud Storage](/googlecloudstorage/)
  * [Google Drive](/drive/)
  * [Google Photos](/googlephotos/)
+  * [HDFS](/hdfs/)
  * [HTTP](/http/)
  * [Hubic](/hubic/)
  * [Jottacloud / GetSky.no](/jottacloud/)
--- a/docs/content/hdfs.md
+++ b/docs/content/hdfs.md
@ -0,0 +1,199 @@
+---
+title: "HDFS Remote"
+description: "Remote for Hadoop Distributed Filesystem"
+---
+
+{{< icon "fa fa-globe" >}} HDFS
+-------------------------------------------------
+
+[HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html) is a
+distributed file-system, part of the [Apache Hadoop](https://hadoop.apache.org/) framework.
+
+Paths are specified as `remote:` or `remote:path/to/dir`.
+
+Here is an example of how to make a remote called `remote`. First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+[skip]
+XX / Hadoop distributed file system
+   \ "hdfs"
+[skip]
+Storage> hdfs
+** See help for hdfs backend at: https://rclone.org/hdfs/ **
+
+hadoop name node and port
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Connect to host namenode at port 8020
+   \ "namenode:8020"
+namenode> namenode.hadoop:8020
+hadoop user name
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Connect to hdfs as root
+   \ "root"
+username> root
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[remote]
+type = hdfs
+namenode = namenode.hadoop:8020
+username = root
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+hadoop               hdfs
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
+```
+
+This remote is called `remote` and can now be used like this
+
+See all the top level directories
+
+    rclone lsd remote:
+
+List the contents of a directory
+
+    rclone ls remote:directory
+
+Sync the remote `directory` to `/home/local/directory`, deleting any excess files.
+
+    rclone sync -i remote:directory /home/local/directory
+
+### Setting up your own HDFS instance for testing
+
+You may start with a [manual setup](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html)
+or use the docker image from the tests:
+
+If you want to build the docker image
+
+```
+git clone https://github.com/rclone/rclone.git
+cd rclone/fstest/testserver/images/test-hdfs
+docker build --rm -t rclone/test-hdfs .
+```
+
+Or you can just use the latest one pushed
+
+```
+docker run --rm --name "rclone-hdfs" -p 127.0.0.1:9866:9866 -p 127.0.0.1:8020:8020 --hostname "rclone-hdfs" rclone/test-hdfs
+```
+
+**NB** it need few seconds to startup.
+
+For this docker image the remote needs to be configured like this:
+
+```
+[remote]
+type = hdfs
+namenode = 127.0.0.1:8020
+username = root
+```
+
+You can stop this image with `docker kill rclone-hdfs` (**NB** it does not use volumes, so all data
+uploaded will be lost.)
+
+### Modified time
+
+Time accurate to 1 second is stored.
+
+### Checksum
+
+No checksums are implemented.
+
+### Usage information
+
+You can use the `rclone about remote:` command which will display filesystem size and current usage.
+
+### Restricted filename characters
+
+In addition to the [default restricted characters set](/overview/#restricted-characters)
+the following characters are also replaced:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| :         | 0x3A  | ：           |
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8).
+
+### Limitations
+
+- No server-side `Move` or `DirMove`.
+- Checksums not implemented.
+
+{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hdfs/hdfs.go then run make backenddocs" >}}
+### Standard Options
+
+Here are the standard options specific to hdfs (Hadoop distributed file system).
+
+#### --hdfs-namenode
+
+hadoop name node and port
+
+- Config:      namenode
+- Env Var:     RCLONE_HDFS_NAMENODE
+- Type:        string
+- Default:     ""
+- Examples:
+    - "namenode:8020"
+        - Connect to host namenode at port 8020
+
+#### --hdfs-username
+
+hadoop user name
+
+- Config:      username
+- Env Var:     RCLONE_HDFS_USERNAME
+- Type:        string
+- Default:     ""
+- Examples:
+    - "root"
+        - Connect to hdfs as root
+
+### Advanced Options
+
+Here are the advanced options specific to hdfs (Hadoop distributed file system).
+
+#### --hdfs-encoding
+
+This sets the encoding for the backend.
+
+See: the [encoding section in the overview](/overview/#encoding) for more info.
+
+- Config:      encoding
+- Env Var:     RCLONE_HDFS_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,Colon,Del,Ctl,InvalidUtf8,Dot
+
+{{< rem autogenerated options stop >}}
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@ -28,6 +28,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Cloud Storage         | MD5         | Yes     | No               | No              | R/W       |
 | Google Drive                 | MD5         | Yes     | No               | Yes             | R/W       |
 | Google Photos                | -           | No      | No               | Yes             | R         |
+| HDFS                         | -           | Yes     | No               | No              | -         |
 | HTTP                         | -           | No      | No               | No              | R         |
 | Hubic                        | MD5         | Yes     | No               | No              | R/W       |
 | Jottacloud                   | MD5         | Yes     | Yes              | No              | R         |
@ -341,6 +342,7 @@ upon backend specific capabilities.
 | Google Cloud Storage         | Yes   | Yes  | No   | No      | No      | Yes   | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | No  | No |
 | Google Drive                 | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | Yes         | Yes | Yes |
 | Google Photos                | No    | No   | No   | No      | No      | No    | No           | No          | No | No |
+| HDFS                         | Yes   | No   | No   | No      | No      | No    | Yes          | No          | Yes | Yes |
 | HTTP                         | No    | No   | No   | No      | No      | No    | No           | No [#2178](https://github.com/rclone/rclone/issues/2178) | No  | Yes |
 | Hubic                        | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | No |
 | Jottacloud                   | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | No           | Yes                                                   | Yes | Yes |
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@ -78,6 +78,7 @@
          <a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google"></i> Google Cloud Storage</a>
          <a class="dropdown-item" href="/drive/"><i class="fab fa-google"></i> Google Drive</a>
          <a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images"></i> Google Photos</a>
+          <a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
          <a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
          <a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
          <a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@ -297,3 +297,8 @@ backends:
 - backend:  "zoho"
   remote:   "TestZoho:"
   fastlist: false
+ - backend: "hdfs"
+   remote: "TestHdfs:"
+   fastlist: false
+   ignore:
+     - TestSyncUTFNorm
--- a/fstest/testserver/images/test-hdfs/Dockerfile
+++ b/fstest/testserver/images/test-hdfs/Dockerfile
@ -0,0 +1,42 @@
+# A very minimal hdfs server for integration testing rclone
+FROM debian:stretch
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl python
+
+ENV HADOOP_VERSION 3.2.1
+ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+RUN set -x \
+    && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \
+    && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \
+    && rm /tmp/hadoop.tar.gz*
+
+RUN ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop
+RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
+
+RUN mkdir /hadoop-data
+RUN mkdir -p /hadoop/dfs/name
+RUN mkdir -p /hadoop/dfs/data
+
+ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
+ENV HADOOP_CONF_DIR=/etc/hadoop
+ENV MULTIHOMED_NETWORK=1
+
+ENV USER=root
+ENV PATH $HADOOP_HOME/bin/:$PATH
+
+ADD core-site.xml    /etc/hadoop/core-site.xml
+ADD hdfs-site.xml    /etc/hadoop/hdfs-site.xml
+ADD httpfs-site.xml  /etc/hadoop/httpfs-site.xml
+ADD kms-site.xml     /etc/hadoop/kms-site.xml
+ADD mapred-site.xml  /etc/hadoop/mapred-site.xml
+ADD yarn-site.xml    /etc/hadoop/yarn-site.xml
+
+ADD run.sh /run.sh
+RUN chmod a+x /run.sh
+CMD ["/run.sh"]
--- a/fstest/testserver/images/test-hdfs/README.md
+++ b/fstest/testserver/images/test-hdfs/README.md
@ -0,0 +1,32 @@
+# Test HDFS
+
+This is a docker image for rclone's integration tests which runs an
+hdfs filesystem in a docker image.
+
+## Build
+
+```
+docker build --rm -t rclone/test-hdfs .
+docker push rclone/test-hdfs
+```
+
+# Test
+
+configure remote:
+```
+[TestHdfs]
+type = hdfs
+namenode = 127.0.0.1:8020
+username = root
+```
+
+run tests
+```
+cd backend/hdfs
+GO111MODULE=on go test -v
+```
+
+stop docker image:
+```
+docker kill rclone-hdfs
+```
--- a/fstest/testserver/images/test-hdfs/core-site.xml
+++ b/fstest/testserver/images/test-hdfs/core-site.xml
@ -0,0 +1,6 @@
+<configuration>
+    <property><name>fs.defaultFS</name><value>hdfs://localhost:8020</value></property>
+    <property><name>hadoop.http.staticuser.user</name><value>root</value></property>
+    <property><name>hadoop.proxyuser.root.groups</name><value>root,nogroup</value></property>
+    <property><name>hadoop.proxyuser.root.hosts</name><value>*</value></property>
+</configuration>
--- a/fstest/testserver/images/test-hdfs/hdfs-site.xml
+++ b/fstest/testserver/images/test-hdfs/hdfs-site.xml
@ -0,0 +1,14 @@
+<configuration>
+    <property><name>dfs.client.use.datanode.hostname</name><value>true</value></property>
+    <property><name>dfs.datanode.data.dir</name><value>file:///hadoop/dfs/data</value></property>
+    <property><name>dfs.datanode.use.datanode.hostname</name><value>true</value></property>
+    <property><name>dfs.namenode.accesstime.precision</name><value>3600000</value></property>
+    <property><name>dfs.namenode.http-bind-host</name><value>0.0.0.0</value></property>
+    <property><name>dfs.namenode.https-bind-host</name><value>0.0.0.0</value></property>
+    <property><name>dfs.namenode.name.dir</name><value>file:///hadoop/dfs/name</value></property>
+    <property><name>dfs.namenode.rpc-bind-host</name><value>0.0.0.0</value></property>
+    <property><name>dfs.namenode.safemode.extension</name><value>5000</value></property>
+    <property><name>dfs.namenode.servicerpc-bind-host</name><value>0.0.0.0</value></property>
+    <property><name>dfs.replication</name><value>2</value></property>
+    <property><name>nfs.dump.dir</name><value>/tmp</value></property>
+</configuration>
--- a/fstest/testserver/images/test-hdfs/httpfs-site.xml
+++ b/fstest/testserver/images/test-hdfs/httpfs-site.xml
@ -0,0 +1,2 @@
+<configuration>
+</configuration>
--- a/fstest/testserver/images/test-hdfs/kms-site.xml
+++ b/fstest/testserver/images/test-hdfs/kms-site.xml
@ -0,0 +1,2 @@
+<configuration>
+</configuration>
--- a/fstest/testserver/images/test-hdfs/mapred-site.xml
+++ b/fstest/testserver/images/test-hdfs/mapred-site.xml
@ -0,0 +1,5 @@
+<configuration>
+	<property><name>mapreduce.framework.name</name><value>yarn</value></property>
+	<property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property>
+</configuration>
+
--- a/fstest/testserver/images/test-hdfs/run.sh
+++ b/fstest/testserver/images/test-hdfs/run.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo format namenode
+hdfs namenode -format test
+
+hdfs namenode &
+hdfs datanode &
+exec sleep infinity
--- a/fstest/testserver/images/test-hdfs/yarn-site.xml
+++ b/fstest/testserver/images/test-hdfs/yarn-site.xml
@ -0,0 +1,14 @@
+<configuration>
+    <property><name>yarn.log-aggregation-enable</name><value>true</value></property>
+    <property><name>yarn.log.server.url</name><value>http://localhost:8188/applicationhistory/logs/</value></property>
+    <property><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>
+    <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property>
+    <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property>
+    <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property>
+    <property><name>yarn.nodemanager.remote-app-log-dir</name><value>/app-logs</value></property>
+    <property><name>yarn.timeline-service.bind-host</name><value>0.0.0.0</value></property>
+    <property><name>yarn.timeline-service.enabled</name><value>true</value></property>
+    <property><name>yarn.timeline-service.generic-application-history.enabled</name><value>true</value></property>
+    <property><name>yarn.timeline-service.hostname</name><value>historyserver.hadoop</value></property>
+    <property><name>yarn.timeline-service.leveldb-timeline-store.path</name><value>/hadoop/yarn/timeline</value></property>
+</configuration>
--- a/fstest/testserver/init.d/TestHdfs
+++ b/fstest/testserver/init.d/TestHdfs
@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+NAME=rclone-hdfs
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name "rclone-hdfs" -p 127.0.0.1:9866:9866 -p 127.0.0.1:8020:8020 --hostname "rclone-hdfs" rclone/test-hdfs
+    sleep 10
+
+    echo type=hdfs
+    echo namenode=127.0.0.1:8020
+    echo user=root
+}
+stop() {
+    if status ; then
+        docker kill $NAME
+        echo "$NAME stopped"
+    fi
+}
+
+. $(dirname "$0")/run.bash
--- a/go.mod
+++ b/go.mod
@ -18,6 +18,7 @@ require (
 	github.com/billziss-gh/cgofuse v1.4.0
 	github.com/buengese/sgzip v0.1.0
 	github.com/calebcase/tmpfile v1.0.2 // indirect
+	github.com/colinmarc/hdfs/v2 v2.1.1
 	github.com/coreos/go-semver v0.3.0
 	github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible
 	github.com/gabriel-vasile/mimetype v1.1.1
--- a/go.sum
+++ b/go.sum
@ -140,6 +140,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/colinmarc/hdfs/v2 v2.1.1 h1:x0hw/m+o3UE20Scso/KCkvYNc9Di39TBlCfGMkJ1/a0=
+github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@ -222,6 +224,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -303,7 +306,9 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@ -325,6 +330,8 @@ github.com/iguanesolutions/go-systemd/v5 v5.0.0/go.mod h1:VPlzL6z0rXd3HU7oLkMoEq
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
+github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930 h1:v4CYlQ+HeysPHsr2QFiEO60gKqnvn1xwvuKhhAhuEkk=
+github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 h1:ly2C51IMpCCV8RpTDRXgzG/L9iZXb8ePEixaew/HwBs=
@ -458,6 +465,7 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
@ -649,6 +657,7 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 goftp.io/server v0.4.0 h1:hqsVdwd1/l6QtYxD9pxca9mEAJYZ7+FPCnmeXKXHQNw=
 goftp.io/server v0.4.0/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -663,7 +672,9 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -738,6 +749,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201029055024-942e2f445f3c h1:rpcgRPA7OvNEOdprt2Wx8/Re2cBTd8NPo/lvo3AyMqk=
 golang.org/x/net v0.0.0-20201029055024-942e2f445f3c/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@ -808,6 +820,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
 golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -902,6 +915,7 @@ google.golang.org/api v0.28.0 h1:jMF5hhVfMkTZwHW1SDpKq5CkgWLXOb31Foaca9Zr3oM=
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.33.0 h1:+gL0XvACeMIvpwLZ5rQZzLn5cwOsgg8dIcfJ2SYfBVw=
 google.golang.org/api v0.33.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.34.0 h1:k40adF3uR+6x/+hO5Dh4ZFUqFp67vxvbpafFiJxl10A=
 google.golang.org/api v0.34.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
@ -948,6 +962,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 h1:bFFRpT+e8JJVY7lMMfvezL1ZIwqiwmPl2bsE2yx4HqM=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201028140639-c77dae4b0522 h1:7RoRaOmOAXwqnurgQ5g5/d0yCi9ha2UxuTZULXudK7A=
 google.golang.org/genproto v0.0.0-20201028140639-c77dae4b0522/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@ -971,6 +986,7 @@ google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
 google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1 h1:DGeFlSan2f+WEtCERJ4J9GJWk15TxUi8QGagfI87Xyc=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
@ -1001,6 +1017,16 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4=
+gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=