serve nfs: new `serve nfs` command

Summary:
Adding a new command to serve any remote over NFS. This is only useful for new macOS versions where FUSE mounts are not available.
 * Added willscot/go-nfs dependency and updated go.mod and go.sum

Test Plan:
```
go run rclone.go serve nfs --http-url https://beta.rclone.org :http:
```

Test that it is serving correctly by mounting the NFS directory.

```
mkdir nfs-test
mount -oport=58654,mountport=58654 localhost: nfs-test
```

Then we can list the mounted directory to see it is working.
```
ls nfs-test
```
This commit is contained in:
Saleh Dindar 2023-10-04 10:28:41 -07:00 committed by Nick Craig-Wood
parent 25f59b2918
commit c69cf46f06
8 changed files with 418 additions and 0 deletions

159
cmd/serve/nfs/filesystem.go Normal file
View File

@ -0,0 +1,159 @@
//go:build unix
// +build unix
package nfs
import (
"os"
"path"
"strings"
"time"
billy "github.com/go-git/go-billy/v5"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
// FS is our wrapper around the VFS to properly support billy.Filesystem interface
type FS struct {
vfs *vfs.VFS
}
// ReadDir implements read dir
func (f *FS) ReadDir(path string) (dir []os.FileInfo, err error) {
return f.vfs.ReadDir(path)
}
// Create implements creating new files
func (f *FS) Create(filename string) (billy.File, error) {
return f.vfs.Create(filename)
}
// Open opens a file
func (f *FS) Open(filename string) (billy.File, error) {
return f.vfs.Open(filename)
}
// OpenFile opens a file
func (f *FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) {
return f.vfs.OpenFile(filename, flag, perm)
}
// Stat gets the file stat
func (f *FS) Stat(filename string) (os.FileInfo, error) {
return f.vfs.Stat(filename)
}
// Rename renames a file
func (f *FS) Rename(oldpath, newpath string) error {
return f.vfs.Rename(oldpath, newpath)
}
// Remove deletes a file
func (f *FS) Remove(filename string) error {
return f.vfs.Remove(filename)
}
// Join joins path elements
func (f *FS) Join(elem ...string) string {
return path.Join(elem...)
}
// TempFile is not implemented
func (f *FS) TempFile(dir, prefix string) (billy.File, error) {
return nil, os.ErrInvalid
}
// MkdirAll creates a directory and all the ones above it
// it does not redirect to VFS.MkDirAll because that one doesn't
// honor the permissions
func (f *FS) MkdirAll(filename string, perm os.FileMode) error {
parts := strings.Split(filename, "/")
for i := range parts {
current := strings.Join(parts[:i+1], "/")
_, err := f.Stat(current)
if err == vfs.ENOENT {
err = f.vfs.Mkdir(current, perm)
if err != nil {
return err
}
}
}
return nil
}
// Lstat gets the stats for symlink
func (f *FS) Lstat(filename string) (os.FileInfo, error) {
return f.vfs.Stat(filename)
}
// Symlink is not supported over NFS
func (f *FS) Symlink(target, link string) error {
return os.ErrInvalid
}
// Readlink is not supported
func (f *FS) Readlink(link string) (string, error) {
return "", os.ErrInvalid
}
// Chmod changes the file modes
func (f *FS) Chmod(name string, mode os.FileMode) error {
file, err := f.vfs.Open(name)
if err != nil {
return err
}
defer func() {
if err := file.Close(); err != nil {
fs.Logf(f, "Error while closing file: %e", err)
}
}()
return file.Chmod(mode)
}
// Lchown changes the owner of symlink
func (f *FS) Lchown(name string, uid, gid int) error {
return f.Chown(name, uid, gid)
}
// Chown changes owner of the file
func (f *FS) Chown(name string, uid, gid int) error {
file, err := f.vfs.Open(name)
if err != nil {
return err
}
defer func() {
if err := file.Close(); err != nil {
fs.Logf(f, "Error while closing file: %e", err)
}
}()
return file.Chown(uid, gid)
}
// Chtimes changes the acces time and modified time
func (f *FS) Chtimes(name string, atime time.Time, mtime time.Time) error {
return f.vfs.Chtimes(name, atime, mtime)
}
// Chroot is not supported in VFS
func (f *FS) Chroot(path string) (billy.Filesystem, error) {
return nil, os.ErrInvalid
}
// Root returns the root of a VFS
func (f *FS) Root() string {
return f.vfs.Fs().Root()
}
// Capabilities exports the filesystem capabilities
func (f *FS) Capabilities() billy.Capability {
if f.vfs.Opt.CacheMode == vfscommon.CacheModeOff {
return billy.ReadCapability | billy.SeekCapability
}
return billy.WriteCapability | billy.ReadCapability |
billy.ReadAndWriteCapability | billy.SeekCapability | billy.TruncateCapability
}
// Interface check
var _ billy.Filesystem = (*FS)(nil)

70
cmd/serve/nfs/handler.go Normal file
View File

@ -0,0 +1,70 @@
//go:build unix
// +build unix
package nfs
import (
"context"
"net"
"github.com/go-git/go-billy/v5"
"github.com/rclone/rclone/vfs"
"github.com/willscott/go-nfs"
nfshelper "github.com/willscott/go-nfs/helpers"
)
// NewBackendAuthHandler creates a handler for the provided filesystem
func NewBackendAuthHandler(vfs *vfs.VFS) nfs.Handler {
return &BackendAuthHandler{vfs}
}
// BackendAuthHandler returns a NFS backing that exposes a given file system in response to all mount requests.
type BackendAuthHandler struct {
vfs *vfs.VFS
}
// Mount backs Mount RPC Requests, allowing for access control policies.
func (h *BackendAuthHandler) Mount(ctx context.Context, conn net.Conn, req nfs.MountRequest) (status nfs.MountStatus, hndl billy.Filesystem, auths []nfs.AuthFlavor) {
status = nfs.MountStatusOk
hndl = &FS{vfs: h.vfs}
auths = []nfs.AuthFlavor{nfs.AuthFlavorNull}
return
}
// Change provides an interface for updating file attributes.
func (h *BackendAuthHandler) Change(fs billy.Filesystem) billy.Change {
if c, ok := fs.(billy.Change); ok {
return c
}
return nil
}
// FSStat provides information about a filesystem.
func (h *BackendAuthHandler) FSStat(ctx context.Context, f billy.Filesystem, s *nfs.FSStat) error {
total, _, free := h.vfs.Statfs()
s.TotalSize = uint64(total)
s.FreeSize = uint64(free)
s.AvailableSize = uint64(free)
return nil
}
// ToHandle handled by CachingHandler
func (h *BackendAuthHandler) ToHandle(f billy.Filesystem, s []string) []byte {
return []byte{}
}
// FromHandle handled by CachingHandler
func (h *BackendAuthHandler) FromHandle([]byte) (billy.Filesystem, []string, error) {
return nil, []string{}, nil
}
// HandleLimit handled by cachingHandler
func (h *BackendAuthHandler) HandleLimit() int {
return -1
}
func newHandler(vfs *vfs.VFS) nfs.Handler {
handler := NewBackendAuthHandler(vfs)
cacheHelper := nfshelper.NewCachingHandler(handler, 1024)
return cacheHelper
}

97
cmd/serve/nfs/nfs.go Normal file
View File

@ -0,0 +1,97 @@
//go:build unix
// +build unix
// Package nfs implements a server to serve a VFS remote over NFSv3 protocol
//
// There is no authentication available on this server
// and it is served on loopback interface by default.
//
// This is primarily used for mounting a VFS remote
// in macOS, where FUSE-mounting mechanisms are usually not available.
package nfs
import (
"context"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Options contains options for the NFS Server
type Options struct {
ListenAddr string // Port to listen on
}
var opt Options
// AddFlags adds flags for the sftp
func AddFlags(flagSet *pflag.FlagSet, Opt *Options) {
rc.AddOption("nfs", &Opt)
flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to", "")
}
func init() {
vfsflags.AddFlags(Command.Flags())
AddFlags(Command.Flags(), &opt)
}
// Run the command
func Run(command *cobra.Command, args []string) {
var f fs.Fs
cmd.CheckArgs(1, 1, command, args)
f = cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s, err := NewServer(context.Background(), vfs.New(f, &vfsflags.Opt), &opt)
if err != nil {
return err
}
return s.Serve()
})
}
// Command is the definition of the command
var Command = &cobra.Command{
Use: "nfs remote:path",
Short: `Serve the remote as an NFS mount`,
Long: `Create an NFS server that serves the given remote over the network.
The primary purpose for this command is to enable [mount command](/commands/rclone_mount/) on recent macOS versions where
installing FUSE is very cumbersome.
Since this is running on NFSv3, no authentication method is available. Any client
will be able to access the data. To limit access, you can use serve NFS on loopback address
and rely on secure tunnels (such as SSH). For this reason, by default, a random TCP port is chosen and loopback interface is used for the listening address;
meaning that it is only available to the local machine. If you want other machines to access the
NFS mount over local network, you need to specify the listening address and port using ` + "`--addr`" + ` flag.
Modifying files through NFS protocol requires VFS caching. Usually you will need to specify ` + "`--vfs-cache-mode`" + `
in order to be able to write to the mountpoint (full is recommended). If you don't specify VFS cache mode,
the mount will be read-only.
To serve NFS over the network use following command:
rclone serve nfs remote: --addr 0.0.0.0:$PORT --vfs-cache-mode=full
We specify a specific port that we can use in the mount command:
To mount the server under Linux/macOS, use the following command:
mount -oport=$PORT,mountport=$PORT $HOSTNAME: path/to/mountpoint
Where ` + "`$PORT`" + ` is the same port number we used in the serve nfs command.
This feature is only available on Unix platforms.
` + vfs.Help,
Annotations: map[string]string{
"versionIntroduced": "v1.65",
"groups": "Filter",
},
Run: Run,
}

View File

@ -0,0 +1,13 @@
// For unsupported architectures
//go:build !unix
// +build !unix
// Package nfs is not supported on non-Unix platforms
package nfs
import (
"github.com/spf13/cobra"
)
// For unsupported platforms we just put nil
var Command *cobra.Command = nil

61
cmd/serve/nfs/server.go Normal file
View File

@ -0,0 +1,61 @@
//go:build unix
// +build unix
package nfs
import (
"context"
"net"
nfs "github.com/willscott/go-nfs"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
// Server contains everything to run the Server
type Server struct {
opt Options
handler nfs.Handler
ctx context.Context // for global config
listener net.Listener
}
// NewServer creates a new server
func NewServer(ctx context.Context, vfs *vfs.VFS, opt *Options) (s *Server, err error) {
if vfs.Opt.CacheMode == vfscommon.CacheModeOff {
fs.LogPrintf(fs.LogLevelWarning, ctx, "NFS writes don't work without a cache, the filesystem will be served read-only")
}
// Our NFS server doesn't have any authentication, we run it on localhost and random port by default
if opt.ListenAddr == "" {
opt.ListenAddr = "localhost:"
}
s = &Server{
ctx: ctx,
opt: *opt,
}
s.handler = newHandler(vfs)
s.listener, err = net.Listen("tcp", s.opt.ListenAddr)
if err != nil {
fs.Errorf(nil, "NFS server failed to listen: %v\n", err)
}
return
}
// Addr returns the listening address of the server
func (s *Server) Addr() net.Addr {
return s.listener.Addr()
}
// Shutdown stops the server
func (s *Server) Shutdown() error {
return s.listener.Close()
}
// Serve starts the server
func (s *Server) Serve() (err error) {
fs.Logf(nil, "NFS Server running at %s\n", s.listener.Addr())
return nfs.Serve(s.listener, s.handler)
}

View File

@ -9,6 +9,7 @@ import (
"github.com/rclone/rclone/cmd/serve/docker"
"github.com/rclone/rclone/cmd/serve/ftp"
"github.com/rclone/rclone/cmd/serve/http"
"github.com/rclone/rclone/cmd/serve/nfs"
"github.com/rclone/rclone/cmd/serve/restic"
"github.com/rclone/rclone/cmd/serve/sftp"
"github.com/rclone/rclone/cmd/serve/webdav"
@ -35,6 +36,9 @@ func init() {
if docker.Command != nil {
Command.AddCommand(docker.Command)
}
if nfs.Command != nil {
Command.AddCommand(nfs.Command)
}
cmd.Root.AddCommand(Command)
}

4
go.mod
View File

@ -60,6 +60,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.4
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca
github.com/willscott/go-nfs v0.0.0-20230823072803-2b8e63b4d81f
github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0
github.com/xanzy/ssh-agent v0.3.3
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
@ -116,6 +117,7 @@ require (
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@ -139,6 +141,7 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/relvacode/iso8601 v1.3.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
@ -147,6 +150,7 @@ require (
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/zeebo/blake3 v0.2.3 // indirect
github.com/zeebo/errs v1.3.0 // indirect

10
go.sum
View File

@ -300,6 +300,8 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/henrybear327/Proton-API-Bridge v0.0.0-20230908065933-5bfa15b567db h1:EpttN0oeR6BvUPis9+4iSyxIacMHXlaHrD/+IvyKPlc=
github.com/henrybear327/Proton-API-Bridge v0.0.0-20230908065933-5bfa15b567db/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts=
github.com/henrybear327/go-proton-api v0.0.0-20230907193451-e563407504ce h1:n1URi7VYiwX/3akX51keQXi6Huy4lJdVc4biJHYk3iw=
@ -398,6 +400,7 @@ github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzV
github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
@ -436,6 +439,8 @@ github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzX
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko=
github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
@ -507,6 +512,10 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willscott/go-nfs v0.0.0-20230823072803-2b8e63b4d81f h1:AWEJVmIm/bnZHTNJd5GPy5yR5riCaa59FsHbvbcW+fI=
github.com/willscott/go-nfs v0.0.0-20230823072803-2b8e63b4d81f/go.mod h1:1PiZswSlNu7GfHMWekF6rF1I94c8E0p+Cwbz7px1D14=
github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33 h1:Wd8wdpRzPXskyHvZLyw7Wc1fp5oCE2mhBCj7bAiibUs=
github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33/go.mod h1:cOUKSNty+RabZqKhm5yTJT5Vq/Fe83ZRWAJ5Kj8nRes=
github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0 h1:j3un8DqYvvAOqKI5OPz+/RRVhDFipbPKI4t2Uk5RBJw=
github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
@ -531,6 +540,7 @@ github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/zema1/go-nfs-client v0.0.0-20200604081958-0cf942f0e0fe/go.mod h1:im3CVJ32XM3+E+2RhY0sa5IVJVQehUrX0oE1wX4xOwU=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=