local: implement modtime and metadata for directories

A consequence of this is that fs.Directory returned by the local
backend will now have a correct size in (rather than -1). Some tests
depended on this and have been fixed by this commit too.

backend/alias/alias_internal_test.go

@@ -81,10 +81,12 @@ func TestNewFS(t *testing.T) {
 		for i, gotEntry := range gotEntries {
 			what := fmt.Sprintf("%s, entry=%d", what, i)
 			wantEntry := test.entries[i]
+			_, isDir := gotEntry.(fs.Directory)
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
-			_, isDir := gotEntry.(fs.Directory)
+			if !isDir {
+				require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			}
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
 	}

backend/local/local.go

@@ -53,6 +53,8 @@ netbsd, macOS and Solaris. It is **not** supported on Windows yet
 
 User metadata is stored as extended attributes (which may not be
 supported by all file systems) under the "user.*" prefix.
+
+Metadata is supported on files and directories.
 `,
 		},
 		Options: []fs.Option{{
@@ -270,6 +272,11 @@ type Object struct {
 	translatedLink bool // Is this object a translated link
 }
 
+// Directory represents a local filesystem directory
+type Directory struct {
+	Object
+}
+
 // ------------------------------------------------------------
 
 var (
@@ -301,15 +308,20 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
-		CaseInsensitive:         f.caseInsensitive(),
-		CanHaveEmptyDirectories: true,
-		IsLocal:                 true,
-		SlowHash:                true,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
-		FilterAware:             true,
-		PartialUploads:          true,
+		CaseInsensitive:          f.caseInsensitive(),
+		CanHaveEmptyDirectories:  true,
+		IsLocal:                  true,
+		SlowHash:                 true,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		DirModTimeUpdatesOnWrite: true,
+		UserMetadata:             xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:              true,
+		PartialUploads:           true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -453,6 +465,15 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
+// Create new directory object from the info passed in
+func (f *Fs) newDirectory(dir string, fi os.FileInfo) *Directory {
+	o := f.newObject(dir)
+	o.setMetadata(fi)
+	return &Directory{
+		Object: *o,
+	}
+}
+
 // List the objects and directories in dir into entries.  The
 // entries can be returned in any order but should be for a
 // complete directory.
@@ -563,7 +584,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
 				if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
-					d := fs.NewDir(newRemote, fi.ModTime())
+					d := f.newDirectory(newRemote, fi)
 					entries = append(entries, d)
 				}
 			} else {
@@ -653,6 +674,48 @@ func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) e
 	return o.SetModTime(ctx, modTime)
 }
 
+// MkdirMetadata makes the directory passed in as dir.
+//
+// It shouldn't return an error if it already exists.
+//
+// If the metadata is not nil it is set.
+//
+// It returns the directory that was created.
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	// Find and or create the directory
+	localPath := f.localPath(dir)
+	fi, err := f.lstat(localPath)
+	if errors.Is(err, os.ErrNotExist) {
+		err := f.Mkdir(ctx, dir)
+		if err != nil {
+			return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
+		}
+		fi, err = f.lstat(localPath)
+		if err != nil {
+			return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Create directory object
+	d := f.newDirectory(dir, fi)
+
+	// Set metadata on the directory object if provided
+	if metadata != nil {
+		err = d.writeMetadata(metadata)
+		if err != nil {
+			return nil, fmt.Errorf("failed to set metadata on directory: %w", err)
+		}
+		// Re-read info now we have finished setting stuff
+		err = d.lstat()
+		if err != nil {
+			return nil, fmt.Errorf("mkdir metadata: failed to re-read info: %w", err)
+		}
+	}
+	return d, nil
+}
+
 // Rmdir removes the directory
 //
 // If it isn't empty it will return an error
@@ -1473,16 +1536,52 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 	return s
 }
 
+// Items returns the count of items in this directory or this
+// directory and subdirectories if known, -1 for unknown
+func (d *Directory) Items() int64 {
+	return -1
+}
+
+// ID returns the internal ID of this directory if known, or
+// "" otherwise
+func (d *Directory) ID() string {
+	return ""
+}
+
+// SetMetadata sets metadata for a Directory
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	err := d.writeMetadata(metadata)
+	if err != nil {
+		return fmt.Errorf("SetMetadata failed on Directory: %w", err)
+	}
+	// Re-read info now we have finished setting stuff
+	return d.lstat()
+}
+
+// Hash does nothing on a directory
+//
+// This method is implemented with the incorrect type signature to
+// stop the Directory type asserting to fs.Object or fs.ObjectInfo
+func (d *Directory) Hash() {
+	// Does nothing
+}
+
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs             = &Fs{}
-	_ fs.Purger         = &Fs{}
-	_ fs.PutStreamer    = &Fs{}
-	_ fs.Mover          = &Fs{}
-	_ fs.DirMover       = &Fs{}
-	_ fs.Commander      = &Fs{}
-	_ fs.OpenWriterAter = &Fs{}
-	_ fs.DirSetModTimer = &Fs{}
-	_ fs.Object         = &Object{}
-	_ fs.Metadataer     = &Object{}
+	_ fs.Fs              = &Fs{}
+	_ fs.Purger          = &Fs{}
+	_ fs.PutStreamer     = &Fs{}
+	_ fs.Mover           = &Fs{}
+	_ fs.DirMover        = &Fs{}
+	_ fs.Commander       = &Fs{}
+	_ fs.OpenWriterAter  = &Fs{}
+	_ fs.DirSetModTimer  = &Fs{}
+	_ fs.MkdirMetadataer = &Fs{}
+	_ fs.Object          = &Object{}
+	_ fs.Metadataer      = &Object{}
+	_ fs.Directory       = &Directory{}
+	_ fs.SetModTimer     = &Directory{}
+	_ fs.SetMetadataer   = &Directory{}
 )

cmd/lsf/lsf.go

@@ -231,6 +231,10 @@ func Lsf(ctx context.Context, fsrc fs.Fs, out io.Writer) error {
 	}
 
 	return operations.ListJSON(ctx, fsrc, "", &opt, func(item *operations.ListJSONItem) error {
+		// Make size deterministic for tests
+		if item.IsDir {
+			item.Size = -1
+		}
 		_, _ = fmt.Fprintln(out, list.Format(item))
 		return nil
 	})

docs/content/overview.md

@@ -61,7 +61,7 @@ Here is an overview of the major features of each cloud storage system.
 | WebDAV                       | MD5, SHA1 ³       | R ⁴     | Depends          | No              | -         | -        |
 | Yandex Disk                  | MD5               | R/W     | No               | No              | R         | -        |
 | Zoho WorkDrive               | -                 | -       | No               | No              | -         | -        |
-| The local filesystem         | All               | R/W     | Depends          | No              | -         | RWU      |
+| The local filesystem         | All               | DR/W    | Depends          | No              | -         | DRWU     |
 
 ¹ Dropbox supports [its own custom
 hash](https://www.dropbox.com/developers/reference/content-hash).
@@ -115,13 +115,21 @@ systems they must support a common hash type.
 Almost all cloud storage systems store some sort of timestamp
 on objects, but several of them not something that is appropriate
 to use for syncing. E.g. some backends will only write a timestamp
-that represent the time of the upload. To be relevant for syncing
+that represents the time of the upload. To be relevant for syncing
 it should be able to store the modification time of the source
 object. If this is not the case, rclone will only check the file
 size by default, though can be configured to check the file hash
 (with the `--checksum` flag). Ideally it should also be possible to
 change the timestamp of an existing file without having to re-upload it.
 
+| Key | Explanation |
+|-----|-------------|
+| `-` | ModTimes not supported - times likely the upload time |
+| `R` | ModTimes supported on files but can't be changed without re-upload |
+| `R/W` | Read and Write ModTimes fully supported on files |
+| `DR` | ModTimes supported on files and directories but can't be changed without re-upload |
+| `DR/W` | Read and Write ModTimes fully supported on files and directories |
+
 Storage systems with a `-` in the ModTime column, means the
 modification read on objects is not the modification time of the
 file when uploaded. It is most likely the time the file was uploaded,
@@ -143,6 +151,9 @@ in a `mount` will be silently ignored.
 Storage systems with `R/W` (for read/write) in the ModTime column,
 means they do also support modtime-only operations.
 
+Storage systems with `D` in the ModTime column means that the
+following symbols apply to directories as well as files.
+
 ### Case Insensitive ###
 
 If a cloud storage systems is case sensitive then it is possible to
@@ -455,9 +466,12 @@ The levels of metadata support are
 
 | Key | Explanation |
 |-----|-------------|
-| `R` | Read only System Metadata |
-| `RW` | Read and write System Metadata |
-| `RWU` | Read and write System Metadata and read and write User Metadata |
+| `R` | Read only System Metadata on files only|
+| `RW` | Read and write System Metadata on files only|
+| `RWU` | Read and write System Metadata and read and write User Metadata on files only|
+| `DR` | Read only System Metadata on files and directories |
+| `DRW` | Read and write System Metadata on files and directories|
+| `DRWU` | Read and write System Metadata and read and write User Metadata on files and directories |
 
 See [the metadata docs](/docs/#metadata) for more info.
 

fs/operations/rc_test.go

@@ -225,7 +225,7 @@ func TestRcList(t *testing.T) {
 	checkSubdir := func(got *operations.ListJSONItem) {
 		assert.Equal(t, "subdir", got.Path)
 		assert.Equal(t, "subdir", got.Name)
-		assert.Equal(t, int64(-1), got.Size)
+		// assert.Equal(t, int64(-1), got.Size) // size can vary for directories
 		assert.Equal(t, "inode/directory", got.MimeType)
 		assert.Equal(t, true, got.IsDir)
 	}
@@ -298,7 +298,7 @@ func TestRcStat(t *testing.T) {
 		stat := fetch(t, "subdir")
 		assert.Equal(t, "subdir", stat.Path)
 		assert.Equal(t, "subdir", stat.Name)
-		assert.Equal(t, int64(-1), stat.Size)
+		// assert.Equal(t, int64(-1), stat.Size) // size can vary for directories
 		assert.Equal(t, "inode/directory", stat.MimeType)
 		assert.Equal(t, true, stat.IsDir)
 	})