github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-11-21 22:50:16 +01:00
Code Releases Activity

Factor new vfs module out of cmd/mountlib

Browse Source 
This is an OS style file system abstraction with directory caching
used in mount, cmount, serve webdav and serve http.
This commit is contained in:
Nick Craig-Wood 2017-10-28 20:01:34 +01:00
parent 6da6b2556b
commit c1aaff220d
20 changed files with 197 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
cmd/cmount/fs.go
View File

@ -12,8 +12,8 @@ import (
"time"
"github.com/billziss-gh/cgofuse/fuse"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@ -21,7 +21,7 @@ const fhUnset = ^uint64(0)
// FS represents the top level filing system
type FS struct {
FS *mountlib.FS
VFS *vfs.VFS
f fs.Fs
openDirs *openFiles
openFilesWr *openFiles
@ -32,7 +32,7 @@ type FS struct {
// NewFS makes a new FS
func NewFS(f fs.Fs) *FS {
fsys := &FS{
FS: mountlib.NewFS(f),
VFS: vfs.New(f),
f: f,
openDirs: newOpenFiles(0x01),
openFilesWr: newOpenFiles(0x02),
@ -45,7 +45,7 @@ func NewFS(f fs.Fs) *FS {
type openFiles struct {
mu sync.Mutex
mark uint8
nodes []mountlib.Noder
nodes []vfs.Noder
}
func newOpenFiles(mark uint8) *openFiles {
@ -55,11 +55,11 @@ func newOpenFiles(mark uint8) *openFiles {
}
// Open a node returning a file handle
func (of *openFiles) Open(node mountlib.Noder) (fh uint64) {
func (of *openFiles) Open(node vfs.Noder) (fh uint64) {
of.mu.Lock()
defer of.mu.Unlock()
var i int
var oldNode mountlib.Noder
var oldNode vfs.Noder
for i, oldNode = range of.nodes {
if oldNode == nil {
of.nodes[i] = node
@ -78,7 +78,7 @@ func (of *openFiles) InRange(fh uint64) bool {
}
// get the node for fh, call with the lock held
func (of *openFiles) get(fh uint64) (i int, node mountlib.Noder, errc int) {
func (of *openFiles) get(fh uint64) (i int, node vfs.Noder, errc int) {
receivedMark := uint8(fh)
if receivedMark != of.mark {
fs.Debugf(nil, "Bad file handle: bad mark 0x%X != 0x%X: 0x%X", receivedMark, of.mark, fh)
@ -99,7 +99,7 @@ func (of *openFiles) get(fh uint64) (i int, node mountlib.Noder, errc int) {
}
// Get the node for the file handle
func (of *openFiles) Get(fh uint64) (node mountlib.Noder, errc int) {
func (of *openFiles) Get(fh uint64) (node vfs.Noder, errc int) {
of.mu.Lock()
_, node, errc = of.get(fh)
of.mu.Unlock()
@ -118,18 +118,18 @@ func (of *openFiles) Close(fh uint64) (errc int) {
}
// lookup a Node given a path
func (fsys *FS) lookupNode(path string) (node mountlib.Node, errc int) {
node, err := fsys.FS.Lookup(path)
func (fsys *FS) lookupNode(path string) (node vfs.Node, errc int) {
node, err := fsys.VFS.Lookup(path)
return node, translateError(err)
}
// lookup a Dir given a path
func (fsys *FS) lookupDir(path string) (dir *mountlib.Dir, errc int) {
func (fsys *FS) lookupDir(path string) (dir *vfs.Dir, errc int) {
node, errc := fsys.lookupNode(path)
if errc != 0 {
return nil, errc
}
dir, ok := node.(*mountlib.Dir)
dir, ok := node.(*vfs.Dir)
if !ok {
return nil, -fuse.ENOTDIR
}
@ -137,19 +137,19 @@ func (fsys *FS) lookupDir(path string) (dir *mountlib.Dir, errc int) {
}
// lookup a parent Dir given a path returning the dir and the leaf
func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *mountlib.Dir, errc int) {
func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *vfs.Dir, errc int) {
parentDir, leaf := path.Split(filePath)
dir, errc = fsys.lookupDir(parentDir)
return leaf, dir, errc
}
// lookup a File given a path
func (fsys *FS) lookupFile(path string) (file *mountlib.File, errc int) {
func (fsys *FS) lookupFile(path string) (file *vfs.File, errc int) {
node, errc := fsys.lookupNode(path)
if errc != 0 {
return nil, errc
}
file, ok := node.(*mountlib.File)
file, ok := node.(*vfs.File)
if !ok {
return nil, -fuse.EISDIR
}
@ -170,7 +170,7 @@ func (fsys *FS) getOpenFilesFromFh(fh uint64) (of *openFiles, errc int) {
}
// Get the underlying handle from the file handle
func (fsys *FS) getHandleFromFh(fh uint64) (handle mountlib.Noder, errc int) {
func (fsys *FS) getHandleFromFh(fh uint64) (handle vfs.Noder, errc int) {
of, errc := fsys.getOpenFilesFromFh(fh)
if errc != 0 {
return nil, errc
@ -179,11 +179,11 @@ func (fsys *FS) getHandleFromFh(fh uint64) (handle mountlib.Noder, errc int) {
}
// get a node from the path or from the fh if not fhUnset
func (fsys *FS) getNode(path string, fh uint64) (node mountlib.Node, errc int) {
func (fsys *FS) getNode(path string, fh uint64) (node vfs.Node, errc int) {
if fh == fhUnset {
node, errc = fsys.lookupNode(path)
} else {
var n mountlib.Noder
var n vfs.Noder
n, errc = fsys.getHandleFromFh(fh)
if errc == 0 {
node = n.Node()
@ -193,27 +193,27 @@ func (fsys *FS) getNode(path string, fh uint64) (node mountlib.Node, errc int) {
}
// stat fills up the stat block for Node
func (fsys *FS) stat(node mountlib.Node, stat *fuse.Stat_t) (errc int) {
func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
var Size uint64
var Blocks uint64
var modTime time.Time
var Mode os.FileMode
switch x := node.(type) {
case *mountlib.Dir:
case *vfs.Dir:
modTime = x.ModTime()
Mode = mountlib.DirPerms | fuse.S_IFDIR
case *mountlib.File:
Mode = vfs.DirPerms | fuse.S_IFDIR
case *vfs.File:
modTime = x.ModTime()
Size = uint64(x.Size())
Blocks = (Size + 511) / 512
Mode = mountlib.FilePerms | fuse.S_IFREG
Mode = vfs.FilePerms | fuse.S_IFREG
}
//stat.Dev = 1
stat.Ino = node.Inode() // FIXME do we need to set the inode number?
stat.Mode = uint32(Mode)
stat.Nlink = 1
stat.Uid = mountlib.UID
stat.Gid = mountlib.GID
stat.Uid = vfs.UID
stat.Gid = vfs.GID
//stat.Rdev
stat.Size = int64(Size)
t := fuse.NewTimespec(modTime)
@ -275,7 +275,7 @@ func (fsys *FS) Readdir(dirPath string,
return errc
}
dir, ok := node.(*mountlib.Dir)
dir, ok := node.(*vfs.Dir)
if !ok {
return -fuse.ENOTDIR
}
@ -342,7 +342,7 @@ func (fsys *FS) Open(path string, flags int) (errc int, fh uint64) {
}
rdwrMode := flags & fuse.O_ACCMODE
var err error
var handle mountlib.Noder
var handle vfs.Noder
switch {
case rdwrMode == fuse.O_RDONLY:
handle, err = file.OpenRead()
@ -385,7 +385,7 @@ func (fsys *FS) Truncate(path string, size int64, fh uint64) (errc int) {
if errc != 0 {
return errc
}
file, ok := node.(*mountlib.File)
file, ok := node.(*vfs.File)
if !ok {
return -fuse.EIO
}
@ -406,7 +406,7 @@ func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
if errc != 0 {
return errc
}
rfh, ok := handle.(*mountlib.ReadFileHandle)
rfh, ok := handle.(*vfs.ReadFileHandle)
if !ok {
// Can only read from read file handle
return -fuse.EIO
@ -425,7 +425,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
if errc != 0 {
return errc
}
wfh, ok := handle.(*mountlib.WriteFileHandle)
wfh, ok := handle.(*vfs.WriteFileHandle)
if !ok {
// Can only write to write file handle
return -fuse.EIO
@ -446,9 +446,9 @@ func (fsys *FS) Flush(path string, fh uint64) (errc int) {
}
var err error
switch x := handle.(type) {
case *mountlib.ReadFileHandle:
case *vfs.ReadFileHandle:
err = x.Flush()
case *mountlib.WriteFileHandle:
case *vfs.WriteFileHandle:
err = x.Flush()
default:
return -fuse.EIO
@ -470,9 +470,9 @@ func (fsys *FS) Release(path string, fh uint64) (errc int) {
_ = of.Close(fh)
var err error
switch x := handle.(type) {
case *mountlib.ReadFileHandle:
case *vfs.ReadFileHandle:
err = x.Release()
case *mountlib.WriteFileHandle:
case *vfs.WriteFileHandle:
err = x.Release()
default:
return -fuse.EIO
@ -540,9 +540,9 @@ func (fsys *FS) Utimens(path string, tmsp []fuse.Timespec) (errc int) {
}
var err error
switch x := node.(type) {
case *mountlib.Dir:
case *vfs.Dir:
err = x.SetModTime(t)
case *mountlib.File:
case *vfs.File:
err = x.SetModTime(t)
}
return translateError(err)
@ -633,21 +633,21 @@ func translateError(err error) (errc int) {
return 0
}
cause := errors.Cause(err)
if mErr, ok := cause.(mountlib.Error); ok {
if mErr, ok := cause.(vfs.Error); ok {
switch mErr {
case mountlib.OK:
case vfs.OK:
return 0
case mountlib.ENOENT:
case vfs.ENOENT:
return -fuse.ENOENT
case mountlib.ENOTEMPTY:
case vfs.ENOTEMPTY:
return -fuse.ENOTEMPTY
case mountlib.EEXIST:
case vfs.EEXIST:
return -fuse.EEXIST
case mountlib.ESPIPE:
case vfs.ESPIPE:
return -fuse.ESPIPE
case mountlib.EBADF:
case vfs.EBADF:
return -fuse.EBADF
case mountlib.EROFS:
case vfs.EROFS:
return -fuse.EROFS
}
}

7
cmd/cmount/mount.go
View File

@ -19,6 +19,7 @@ import (
"github.com/billziss-gh/cgofuse/fuse"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@ -69,7 +70,7 @@ func mountOptions(device string, mountpoint string) (options []string) {
if mountlib.DefaultPermissions {
options = append(options, "-o", "default_permissions")
}
if mountlib.ReadOnly {
if vfs.ReadOnly {
options = append(options, "-o", "ro")
}
if mountlib.WritebackCache {
@ -90,7 +91,7 @@ func mountOptions(device string, mountpoint string) (options []string) {
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error) {
func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
fs.Debugf(f, "Mounting on %q", mountpoint)
// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
@ -160,7 +161,7 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error
found:
}
return fsys.FS, errChan, unmount, nil
return fsys.VFS, errChan, unmount, nil
}
// Mount mounts the remote at mountpoint.

22
cmd/mount/dir.go
View File

@ -9,8 +9,8 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@ -27,13 +27,7 @@ type DirEntry struct {
// Dir represents a directory entry
type Dir struct {
*mountlib.Dir
// f fs.Fs
// path string
// modTime time.Time
// mu sync.RWMutex // protects the following
// read time.Time // time directory entry last read
// items map[string]*DirEntry
*vfs.Dir
}
// Check interface satsified
@ -42,9 +36,9 @@ var _ fusefs.Node = (*Dir)(nil)
// Attr updates the attributes of a directory
func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
defer fs.Trace(d, "")("attr=%+v, err=%v", a, &err)
a.Gid = mountlib.GID
a.Uid = mountlib.UID
a.Mode = os.ModeDir | mountlib.DirPerms
a.Gid = vfs.GID
a.Uid = vfs.UID
a.Mode = os.ModeDir | vfs.DirPerms
modTime := d.ModTime()
a.Atime = modTime
a.Mtime = modTime
@ -61,7 +55,7 @@ var _ fusefs.NodeSetattrer = (*Dir)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime only.
func (d *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer fs.Trace(d, "stat=%+v", req)("err=%v", &err)
if mountlib.NoModTime {
if vfs.NoModTime {
return nil
}
@ -90,9 +84,9 @@ func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.Lo
return nil, translateError(err)
}
switch x := mnode.(type) {
case *mountlib.File:
case *vfs.File:
return &File{x}, nil
case *mountlib.Dir:
case *vfs.Dir:
return &Dir{x}, nil
}
panic("bad type")

24
cmd/mount/file.go
View File

@ -7,21 +7,15 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// File represents a file
type File struct {
*mountlib.File
// size int64 // size of file - read and written with atomic int64 - must be 64 bit aligned
// d *Dir // parent directory - read only
// mu sync.RWMutex // protects the following
// o fs.Object // NB o may be nil if file is being written
// writers int // number of writers for this file
// pendingModTime time.Time // will be applied once o becomes available, i.e. after file was written
*vfs.File
}
// Check interface satisfied
@ -33,9 +27,9 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
modTime := f.File.ModTime()
Size := uint64(f.File.Size())
Blocks := (Size + 511) / 512
a.Gid = mountlib.GID
a.Uid = mountlib.UID
a.Mode = mountlib.FilePerms
a.Gid = vfs.GID
a.Uid = vfs.UID
a.Mode = vfs.FilePerms
a.Size = Size
a.Atime = modTime
a.Mtime = modTime
@ -51,7 +45,7 @@ var _ fusefs.NodeSetattrer = (*File)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime only.
func (f *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer fs.Trace(f, "a=%+v", req)("err=%v", &err)
if mountlib.NoModTime {
if vfs.NoModTime {
return nil
}
if req.Valid.MtimeNow() {
@ -70,15 +64,15 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
defer fs.Trace(f, "flags=%v", req.Flags)("fh=%v, err=%v", &fh, &err)
switch {
case req.Flags.IsReadOnly():
if mountlib.NoSeek {
if vfs.NoSeek {
resp.Flags |= fuse.OpenNonSeekable
}
var rfh *mountlib.ReadFileHandle
var rfh *vfs.ReadFileHandle
rfh, err = f.File.OpenRead()
fh = &ReadFileHandle{rfh}
case req.Flags.IsWriteOnly() || (req.Flags.IsReadWrite() && (req.Flags&fuse.OpenTruncate) != 0):
resp.Flags |= fuse.OpenNonSeekable
var wfh *mountlib.WriteFileHandle
var wfh *vfs.WriteFileHandle
wfh, err = f.File.OpenWrite()
fh = &WriteFileHandle{wfh}
case req.Flags.IsReadWrite():

26
cmd/mount/fs.go
View File

@ -9,15 +9,15 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// FS represents the top level filing system
type FS struct {
*mountlib.FS
*vfs.VFS
f fs.Fs
}
@ -27,8 +27,8 @@ var _ fusefs.FS = (*FS)(nil)
// NewFS makes a new FS
func NewFS(f fs.Fs) *FS {
fsys := &FS{
FS: mountlib.NewFS(f),
f: f,
VFS: vfs.New(f),
f: f,
}
return fsys
}
@ -36,7 +36,7 @@ func NewFS(f fs.Fs) *FS {
// Root returns the root node
func (f *FS) Root() (node fusefs.Node, err error) {
defer fs.Trace("", "")("node=%+v, err=%v", &node, &err)
root, err := f.FS.Root()
root, err := f.VFS.Root()
if err != nil {
return nil, translateError(err)
}
@ -69,21 +69,21 @@ func translateError(err error) error {
return nil
}
cause := errors.Cause(err)
if mErr, ok := cause.(mountlib.Error); ok {
if mErr, ok := cause.(vfs.Error); ok {
switch mErr {
case mountlib.OK:
case vfs.OK:
return nil
case mountlib.ENOENT:
case vfs.ENOENT:
return fuse.ENOENT
case mountlib.ENOTEMPTY:
case vfs.ENOTEMPTY:
return fuse.Errno(syscall.ENOTEMPTY)
case mountlib.EEXIST:
case vfs.EEXIST:
return fuse.EEXIST
case mountlib.ESPIPE:
case vfs.ESPIPE:
return fuse.Errno(syscall.ESPIPE)
case mountlib.EBADF:
case vfs.EBADF:
return fuse.Errno(syscall.EBADF)
case mountlib.EROFS:
case vfs.EROFS:
return fuse.Errno(syscall.EROFS)
}
}

7
cmd/mount/mount.go
View File

@ -13,6 +13,7 @@ import (
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@ -48,7 +49,7 @@ func mountOptions(device string) (options []fuse.MountOption) {
if mountlib.DefaultPermissions {
options = append(options, fuse.DefaultPermissions())
}
if mountlib.ReadOnly {
if vfs.ReadOnly {
options = append(options, fuse.ReadOnly())
}
if mountlib.WritebackCache {
@ -69,7 +70,7 @@ func mountOptions(device string) (options []fuse.MountOption) {
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error) {
func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
fs.Debugf(f, "Mounting on %q", mountpoint)
c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
if err != nil {
@ -100,7 +101,7 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error
return fuse.Unmount(mountpoint)
}
return filesys.FS, errChan, unmount, nil
return filesys.VFS, errChan, unmount, nil
}
// Mount mounts the remote at mountpoint.

10
cmd/mount/read.go
View File

@ -5,20 +5,14 @@ package mount
import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context"
)
// ReadFileHandle is an open for read file handle on a File
type ReadFileHandle struct {
*mountlib.ReadFileHandle
// mu sync.Mutex
// closed bool // set if handle has been closed
// r *fs.Account
// o fs.Object
// readCalled bool // set if read has been called
// offset int64
*vfs.ReadFileHandle
}
// Check interface satisfied

4
cmd/mount/write.go
View File

@ -7,8 +7,8 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context"
)
@ -16,7 +16,7 @@ var errClosedFileHandle = errors.New("Attempt to use closed file handle")
// WriteFileHandle is an open for write handle on a File
type WriteFileHandle struct {
*mountlib.WriteFileHandle
*vfs.WriteFileHandle
}
// Check interface satisfied

42
cmd/mountlib/mount.go
View File

@ -1,42 +1,25 @@
package mountlib
// Globals
import (
"log"
"os"
"time"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Options set by command line flags
var (
NoModTime = false
NoChecksum = false
DebugFUSE = false
NoSeek = false
DirCacheTime = 5 * 60 * time.Second
PollInterval = time.Minute
// mount options
ReadOnly = false
DebugFUSE = false
AllowNonEmpty = false
AllowRoot = false
AllowOther = false
DefaultPermissions = false
WritebackCache = false
MaxReadAhead fs.SizeSuffix = 128 * 1024
Umask = 0
UID = ^uint32(0) // these values instruct WinFSP-FUSE to use the current user
GID = ^uint32(0) // overriden for non windows in mount_unix.go
// foreground = false
// default permissions for directories - modified by umask in Mount
DirPerms = os.FileMode(0777)
FilePerms = os.FileMode(0666)
ExtraOptions *[]string
ExtraFlags *[]string
ExtraOptions *[]string
ExtraFlags *[]string
)
// NewMountCommand makes a mount command with the given name and Mount function
@ -149,10 +132,6 @@ like this:
cmd.CheckArgs(2, 2, command, args)
fdst := cmd.NewFsDst(args)
// Mask permissions
DirPerms = 0777 &^ os.FileMode(Umask)
FilePerms = 0666 &^ os.FileMode(Umask)
// Show stats if the user has specifically requested them
if cmd.ShowStats() {
stopStats := cmd.StartStats()
@ -184,18 +163,7 @@ like this:
//flags.BoolVarP(&foreground, "foreground", "", foreground, "Do not detach.")
// Add in the generic flags
AddFlags(flags)
vfs.AddFlags(flags)
return commandDefintion
}
// AddFlags adds the non filing system specific flags to the command
func AddFlags(flags *pflag.FlagSet) {
flags.BoolVarP(&NoModTime, "no-modtime", "", NoModTime, "Don't read/write the modification time (can speed things up).")
flags.BoolVarP(&NoChecksum, "no-checksum", "", NoChecksum, "Don't compare checksums on up/download.")
flags.BoolVarP(&NoSeek, "no-seek", "", NoSeek, "Don't allow seeking in files.")
flags.DurationVarP(&DirCacheTime, "dir-cache-time", "", DirCacheTime, "Time to cache directory entries for.")
flags.DurationVarP(&PollInterval, "poll-interval", "", PollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.")
flags.BoolVarP(&ReadOnly, "read-only", "", ReadOnly, "Mount read-only.")
platformFlags(flags)
}

2
cmd/mountlib/mounttest/dir.go
View File

@ -176,7 +176,7 @@ func TestDirCacheFlush(t *testing.T) {
err := run.fremote.Mkdir("dir/subdir")
require.NoError(t, err)
root, err := run.filesys.Root()
root, err := run.vfs.Root()
require.NoError(t, err)
// expect newly created "subdir" on remote to not show up

14
cmd/mountlib/mounttest/fs.go
View File

@ -15,10 +15,10 @@ import (
"testing"
"time"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
_ "github.com/ncw/rclone/fs/all" // import all the file systems
"github.com/ncw/rclone/fstest"
"github.com/ncw/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -27,7 +27,7 @@ type (
// UnmountFn is called to unmount the file system
UnmountFn func() error
// MountFn is called to mount the file system
MountFn func(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error)
MountFn func(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error)
)
var (
@ -46,7 +46,7 @@ func TestMain(m *testing.M, fn MountFn) {
// Run holds the remotes for a test run
type Run struct {
filesys *mountlib.FS
vfs *vfs.VFS
mountPath string
fremote fs.Fs
fremoteName string
@ -112,7 +112,7 @@ func newRun() *Run {
func (r *Run) mount() {
log.Printf("mount %q %q", r.fremote, r.mountPath)
var err error
r.filesys, r.umountResult, r.umountFn, err = mountFn(r.fremote, r.mountPath)
r.vfs, r.umountResult, r.umountFn, err = mountFn(r.fremote, r.mountPath)
if err != nil {
log.Printf("mount failed: %v", err)
r.skip = true
@ -207,10 +207,10 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filepath string) {
if fi.IsDir() {
dir[name+"/"] = struct{}{}
r.readLocal(t, dir, name)
assert.Equal(t, mountlib.DirPerms, fi.Mode().Perm())
assert.Equal(t, vfs.DirPerms, fi.Mode().Perm())
} else {
dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
assert.Equal(t, mountlib.FilePerms, fi.Mode().Perm())
assert.Equal(t, vfs.FilePerms, fi.Mode().Perm())
}
}
}
@ -292,5 +292,5 @@ func TestRoot(t *testing.T) {
fi, err := os.Lstat(run.mountPath)
require.NoError(t, err)
assert.True(t, fi.IsDir())
assert.Equal(t, fi.Mode().Perm(), mountlib.DirPerms)
assert.Equal(t, fi.Mode().Perm(), vfs.DirPerms)
}

2
cmd/mountlib/createinfo.go → vfs/createinfo.go
View File

@ -1,4 +1,4 @@
package mountlib
package vfs
import (
"time"

28
cmd/mountlib/dir.go → vfs/dir.go
View File

@ -1,4 +1,4 @@
package mountlib
package vfs
import (
"os"
@ -14,7 +14,7 @@ import (
// Dir represents a directory entry
type Dir struct {
fsys *FS
vfs *VFS
inode uint64 // inode number
f fs.Fs
parent *Dir // parent, nil for root
@ -26,9 +26,9 @@ type Dir struct {
items map[string]Node // NB can be nil when directory not read yet
}
func newDir(fsys *FS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
return &Dir{
fsys: fsys,
vfs: vfs,
f: f,
parent: parent,
entry: fsDir,
@ -169,7 +169,7 @@ func (d *Dir) _readDir() error {
// fs.Debugf(d.path, "Reading directory")
} else {
age := when.Sub(d.read)
if age < d.fsys.dirCacheTime {
if age < d.vfs.dirCacheTime {
return nil
}
fs.Debugf(d.path, "Re-reading directory (%v old)", age)
@ -208,7 +208,7 @@ func (d *Dir) _readDir() error {
}
}
}
d.items[name] = newDir(d.fsys, d.f, d, dir)
d.items[name] = newDir(d.vfs, d.f, d, dir)
default:
err = errors.Errorf("unknown type %T", item)
fs.Errorf(d.path, "readDir error: %v", err)
@ -260,7 +260,7 @@ func (d *Dir) Size() int64 {
// SetModTime sets the modTime for this dir
func (d *Dir) SetModTime(modTime time.Time) error {
if d.fsys.readOnly {
if d.vfs.readOnly {
return EROFS
}
d.mu.Lock()
@ -309,7 +309,7 @@ func (d *Dir) ReadDirAll() (items Nodes, err error) {
// Create makes a new file
func (d *Dir) Create(name string) (*File, *WriteFileHandle, error) {
if d.fsys.readOnly {
if d.vfs.readOnly {
return nil, nil, EROFS
}
path := path.Join(d.path, name)
@ -328,7 +328,7 @@ func (d *Dir) Create(name string) (*File, *WriteFileHandle, error) {
// Mkdir creates a new directory
func (d *Dir) Mkdir(name string) (*Dir, error) {
if d.fsys.readOnly {
if d.vfs.readOnly {
return nil, EROFS
}
path := path.Join(d.path, name)
@ -339,7 +339,7 @@ func (d *Dir) Mkdir(name string) (*Dir, error) {
return nil, err
}
fsDir := fs.NewDir(path, time.Now())
dir := newDir(d.fsys, d.f, d, fsDir)
dir := newDir(d.vfs, d.f, d, fsDir)
d.addObject(dir)
// fs.Debugf(path, "Dir.Mkdir OK")
return dir, nil
@ -347,7 +347,7 @@ func (d *Dir) Mkdir(name string) (*Dir, error) {
// Remove the directory
func (d *Dir) Remove() error {
if d.fsys.readOnly {
if d.vfs.readOnly {
return EROFS
}
// Check directory is empty first
@ -375,7 +375,7 @@ func (d *Dir) Remove() error {
// RemoveAll removes the directory and any contents recursively
func (d *Dir) RemoveAll() error {
if d.fsys.readOnly {
if d.vfs.readOnly {
return EROFS
}
// Remove contents of the directory
@ -403,7 +403,7 @@ func (d *Dir) DirEntry() (entry fs.DirEntry) {
// which must be a directory. The entry to be removed may correspond
// to a file (unlink) or to a directory (rmdir).
func (d *Dir) RemoveName(name string) error {
if d.fsys.readOnly {
if d.vfs.readOnly {
return EROFS
}
path := path.Join(d.path, name)
@ -418,7 +418,7 @@ func (d *Dir) RemoveName(name string) error {
// Rename the file
func (d *Dir) Rename(oldName, newName string, destDir *Dir) error {
if d.fsys.readOnly {
if d.vfs.readOnly {
return EROFS
}
oldPath := path.Join(d.path, oldName)

2
cmd/mountlib/errors.go → vfs/errors.go
View File

@ -1,6 +1,6 @@
// Cross platform errors
package mountlib
package vfs
import "fmt"

10
cmd/mountlib/file.go → vfs/file.go
View File

@ -1,4 +1,4 @@
package mountlib
package vfs
import (
"os"
@ -98,7 +98,7 @@ func (f *File) ModTime() (modTime time.Time) {
f.mu.Lock()
defer f.mu.Unlock()
if !f.d.fsys.noModTime {
if !f.d.vfs.noModTime {
// if o is nil it isn't valid yet or there are writers, so return the size so far
if f.o == nil || f.writers != 0 {
if !f.pendingModTime.IsZero() {
@ -126,7 +126,7 @@ func (f *File) Size() int64 {
// SetModTime sets the modtime for the file
func (f *File) SetModTime(modTime time.Time) error {
if f.d.fsys.readOnly {
if f.d.vfs.readOnly {
return EROFS
}
f.mu.Lock()
@ -224,7 +224,7 @@ func (f *File) OpenRead() (fh *ReadFileHandle, err error) {
// OpenWrite open the file for write
func (f *File) OpenWrite() (fh *WriteFileHandle, err error) {
if f.d.fsys.readOnly {
if f.d.vfs.readOnly {
return nil, EROFS
}
// if o is nil it isn't valid yet
@ -254,7 +254,7 @@ func (f *File) Fsync() error {
// Remove the file
func (f *File) Remove() error {
if f.d.fsys.readOnly {
if f.d.vfs.readOnly {
return EROFS
}
err := f.o.Remove()

6
cmd/mountlib/read.go → vfs/read.go
View File

@ -1,4 +1,4 @@
package mountlib
package vfs
import (
"io"
@ -34,7 +34,7 @@ var (
func newReadFileHandle(f *File, o fs.Object) (*ReadFileHandle, error) {
var hash *fs.MultiHasher
var err error
if !f.d.fsys.noChecksum {
if !f.d.vfs.noChecksum {
hash, err = fs.NewMultiHasherTypes(o.Fs().Hashes())
if err != nil {
fs.Errorf(o.Fs(), "newReadFileHandle hash error: %v", err)
@ -43,7 +43,7 @@ func newReadFileHandle(f *File, o fs.Object) (*ReadFileHandle, error) {
fh := &ReadFileHandle{
o: o,
noSeek: f.d.fsys.noSeek,
noSeek: f.d.vfs.noSeek,
file: f,
hash: hash,
}

97
cmd/mountlib/fs.go → vfs/vfs.go
View File

@ -1,4 +1,13 @@
package mountlib
// Package vfs provides a virtual filing system layer over rclone's
// native objects.
//
// It attempts to behave in a similar way to Go's filing system
// manipulation code in the os package. The same named function
// should behave in an identical fashion. The objects also obey Go's
// standard interfaces.
//
// It also includes directory caching
package vfs
import (
"fmt"
@ -8,9 +17,28 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/spf13/pflag"
)
// Node represents either a *Dir or a *File
// Options set by command line flags
var (
NoModTime = false
NoChecksum = false
NoSeek = false
DirCacheTime = 5 * 60 * time.Second
PollInterval = time.Minute
// mount options
ReadOnly = false
Umask = 0
UID = ^uint32(0) // these values instruct WinFSP-FUSE to use the current user
GID = ^uint32(0) // overriden for non windows in mount_unix.go
// foreground = false
// default permissions for directories - modified by umask in New
DirPerms = os.FileMode(0777)
FilePerms = os.FileMode(0666)
)
// Node represents either a directory (*Dir) or a file (*File)
type Node interface {
os.FileInfo
IsFile() bool
@ -22,6 +50,7 @@ type Node interface {
DirEntry() fs.DirEntry
}
// Check interfaces
var (
_ Node = (*File)(nil)
_ Node = (*Dir)(nil)
@ -41,6 +70,7 @@ type Noder interface {
Node() Node
}
// Check interfaces
var (
_ Noder = (*File)(nil)
_ Noder = (*Dir)(nil)
@ -48,61 +78,65 @@ var (
_ Noder = (*WriteFileHandle)(nil)
)
// FS represents the top level filing system
type FS struct {
// VFS represents the top level filing system
type VFS struct {
f fs.Fs
root *Dir
noSeek bool // don't allow seeking if set
noChecksum bool // don't check checksums if set
readOnly bool // if set FS is read only
readOnly bool // if set VFS is read only
noModTime bool // don't read mod times for files
dirCacheTime time.Duration // how long to consider directory listing cache valid
}
// NewFS creates a new filing system and root directory
func NewFS(f fs.Fs) *FS {
// New creates a new VFS and root directory
func New(f fs.Fs) *VFS {
fsDir := fs.NewDir("", time.Now())
fsys := &FS{
vfs := &VFS{
f: f,
}
// Mask permissions
DirPerms = 0777 &^ os.FileMode(Umask)
FilePerms = 0666 &^ os.FileMode(Umask)
if NoSeek {
fsys.noSeek = true
vfs.noSeek = true
}
if NoChecksum {
fsys.noChecksum = true
vfs.noChecksum = true
}
if ReadOnly {
fsys.readOnly = true
vfs.readOnly = true
}
if NoModTime {
fsys.noModTime = true
vfs.noModTime = true
}
fsys.dirCacheTime = DirCacheTime
vfs.dirCacheTime = DirCacheTime
fsys.root = newDir(fsys, f, nil, fsDir)
vfs.root = newDir(vfs, f, nil, fsDir)
if PollInterval > 0 {
fsys.PollChanges(PollInterval)
vfs.PollChanges(PollInterval)
}
return fsys
return vfs
}
// PollChanges will poll the remote every pollInterval for changes if the remote
// supports it. If a non-polling option is used, the given time interval can be
// ignored
func (fsys *FS) PollChanges(pollInterval time.Duration) *FS {
doDirChangeNotify := fsys.f.Features().DirChangeNotify
func (vfs *VFS) PollChanges(pollInterval time.Duration) *VFS {
doDirChangeNotify := vfs.f.Features().DirChangeNotify
if doDirChangeNotify != nil {
doDirChangeNotify(fsys.root.ForgetPath, pollInterval)
doDirChangeNotify(vfs.root.ForgetPath, pollInterval)
}
return fsys
return vfs
}
// Root returns the root node
func (fsys *FS) Root() (*Dir, error) {
// fs.Debugf(fsys.f, "Root()")
return fsys.root, nil
func (vfs *VFS) Root() (*Dir, error) {
// fs.Debugf(vfs.f, "Root()")
return vfs.root, nil
}
var inodeCount uint64
@ -113,8 +147,8 @@ func NewInode() (inode uint64) {
}
// Lookup finds the Node by path starting from the root
func (fsys *FS) Lookup(path string) (node Node, err error) {
node = fsys.root
func (vfs *VFS) Lookup(path string) (node Node, err error) {
node = vfs.root
for path != "" {
i := strings.IndexRune(path, '/')
var name string
@ -141,7 +175,7 @@ func (fsys *FS) Lookup(path string) (node Node, err error) {
// Statfs is called to obtain file system metadata.
// It should write that data to resp.
func (fsys *FS) Statfs() error {
func (vfs *VFS) Statfs() error {
/* FIXME
const blockSize = 4096
const fsBlocks = (1 << 50) / blockSize
@ -156,3 +190,14 @@ func (fsys *FS) Statfs() error {
*/
return nil
}
// AddFlags adds the non filing system specific flags to the command
func AddFlags(flags *pflag.FlagSet) {
flags.BoolVarP(&NoModTime, "no-modtime", "", NoModTime, "Don't read/write the modification time (can speed things up).")
flags.BoolVarP(&NoChecksum, "no-checksum", "", NoChecksum, "Don't compare checksums on up/download.")
flags.BoolVarP(&NoSeek, "no-seek", "", NoSeek, "Don't allow seeking in files.")
flags.DurationVarP(&DirCacheTime, "dir-cache-time", "", DirCacheTime, "Time to cache directory entries for.")
flags.DurationVarP(&PollInterval, "poll-interval", "", PollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.")
flags.BoolVarP(&ReadOnly, "read-only", "", ReadOnly, "Mount read-only.")
platformFlags(flags)
}

2
cmd/mountlib/mount_non_unix.go → vfs/vfs_non_unix.go
View File

@ -1,6 +1,6 @@
// +build !linux,!darwin,!freebsd
package mountlib
package vfs
import (
"github.com/spf13/pflag"

2
cmd/mountlib/mount_unix.go → vfs/vfs_unix.go
View File

@ -1,6 +1,6 @@
// +build linux darwin freebsd
package mountlib
package vfs
import (
"github.com/spf13/pflag"

2
cmd/mountlib/write.go → vfs/write.go
View File

@ -1,4 +1,4 @@
package mountlib
package vfs
import (
"io"