mirror of https://github.com/rclone/rclone
serve nbd: serve an rclone remote as a Network Block Device - WIP FIXME
TODO - Need to finalise rclone/gonbdserver and upload and change go.mod/go.sum Maybe make companion `mount nbd` command? Fixes #7337
This commit is contained in:
Nick Craig-Wood
parent
e00bad07e6
commit
265b5eb1f1
|
|
@ -0,0 +1,140 @@
|
|||
// Implements an nbd.Backend for serving from a chunked file in the VFS.
|
||||
|
||||
package nbd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/rclone/gonbdserver/nbd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/vfs/chunked"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// Backend for a single chunked file
|
||||
type chunkedBackend struct {
|
||||
file *chunked.File
|
||||
ec *nbd.ExportConfig
|
||||
}
|
||||
|
||||
// Create Backend for a single chunked file
|
||||
type chunkedBackendFactory struct {
|
||||
s *NBD
|
||||
file *chunked.File
|
||||
}
|
||||
|
||||
// WriteAt implements Backend.WriteAt
|
||||
func (cb *chunkedBackend) WriteAt(ctx context.Context, b []byte, offset int64, fua bool) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", len(b), offset)("n=%d, err=%v", &n, &err)
|
||||
n, err = cb.file.WriteAt(b, offset)
|
||||
if err != nil || !fua {
|
||||
return n, err
|
||||
}
|
||||
err = cb.file.Sync()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadAt implements Backend.ReadAt
|
||||
func (cb *chunkedBackend) ReadAt(ctx context.Context, b []byte, offset int64) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", len(b), offset)("n=%d, err=%v", &n, &err)
|
||||
return cb.file.ReadAt(b, offset)
|
||||
}
|
||||
|
||||
// TrimAt implements Backend.TrimAt
|
||||
func (cb *chunkedBackend) TrimAt(ctx context.Context, length int, offset int64) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", length, offset)("n=%d, err=%v", &n, &err)
|
||||
return length, nil
|
||||
}
|
||||
|
||||
// Flush implements Backend.Flush
|
||||
func (cb *chunkedBackend) Flush(ctx context.Context) (err error) {
|
||||
defer log.Trace(logPrefix, "")("err=%v", &err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements Backend.Close
|
||||
func (cb *chunkedBackend) Close(ctx context.Context) (err error) {
|
||||
defer log.Trace(logPrefix, "")("err=%v", &err)
|
||||
err = cb.file.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Geometry implements Backend.Geometry
|
||||
func (cb *chunkedBackend) Geometry(ctx context.Context) (size uint64, minBS uint64, prefBS uint64, maxBS uint64, err error) {
|
||||
defer log.Trace(logPrefix, "")("size=%d, minBS=%d, prefBS=%d, maxBS=%d, err=%v", &size, &minBS, &prefBS, &maxBS, &err)
|
||||
size = uint64(cb.file.Size())
|
||||
minBS = cb.ec.MinimumBlockSize
|
||||
prefBS = cb.ec.PreferredBlockSize
|
||||
maxBS = cb.ec.MaximumBlockSize
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
|
||||
// HasFua implements Backend.HasFua
|
||||
func (cb *chunkedBackend) HasFua(ctx context.Context) (fua bool) {
|
||||
defer log.Trace(logPrefix, "")("fua=%v", &fua)
|
||||
return true
|
||||
}
|
||||
|
||||
// HasFlush implements Backend.HasFua
|
||||
func (cb *chunkedBackend) HasFlush(ctx context.Context) (flush bool) {
|
||||
defer log.Trace(logPrefix, "")("flush=%v", &flush)
|
||||
return true
|
||||
}
|
||||
|
||||
// New generates a new chunked backend
|
||||
func (cbf *chunkedBackendFactory) newBackend(ctx context.Context, ec *nbd.ExportConfig) (nbd.Backend, error) {
|
||||
err := cbf.file.Open(false, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open chunked file: %w", err)
|
||||
}
|
||||
cb := &chunkedBackend{
|
||||
file: cbf.file,
|
||||
ec: ec,
|
||||
}
|
||||
return cb, nil
|
||||
}
|
||||
|
||||
// Generate a chunked backend factory
|
||||
func (s *NBD) newChunkedBackendFactory(ctx context.Context) (bf backendFactory, err error) {
|
||||
create := s.opt.Create != 0
|
||||
if s.vfs.Opt.ReadOnly && create {
|
||||
return nil, errors.New("can't create files with --read-only")
|
||||
}
|
||||
file := chunked.New(s.vfs, s.leaf)
|
||||
err = file.Open(create, s.opt.Log2ChunkSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open chunked file: %w", err)
|
||||
}
|
||||
defer fs.CheckClose(file, &err)
|
||||
var truncateSize fs.SizeSuffix
|
||||
if create {
|
||||
if file.Size() == 0 {
|
||||
truncateSize = s.opt.Create
|
||||
}
|
||||
} else {
|
||||
truncateSize = s.opt.Resize
|
||||
}
|
||||
if truncateSize != 0 {
|
||||
err = file.Truncate(int64(truncateSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create chunked file: %w", err)
|
||||
}
|
||||
fs.Logf(logPrefix, "Size of network block device is now %v", truncateSize)
|
||||
}
|
||||
return &chunkedBackendFactory{
|
||||
s: s,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Check interfaces
|
||||
var (
|
||||
_ nbd.Backend = (*chunkedBackend)(nil)
|
||||
_ backendFactory = (*chunkedBackendFactory)(nil)
|
||||
)
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
// Implements an nbd.Backend for serving from the VFS.
|
||||
|
||||
package nbd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/rclone/gonbdserver/nbd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// Backend for a single file
|
||||
type fileBackend struct {
|
||||
file vfs.Handle
|
||||
ec *nbd.ExportConfig
|
||||
}
|
||||
|
||||
// Create Backend for a single file
|
||||
type fileBackendFactory struct {
|
||||
s *NBD
|
||||
vfs *vfs.VFS
|
||||
filePath string
|
||||
perms int
|
||||
}
|
||||
|
||||
// WriteAt implements Backend.WriteAt
|
||||
func (fb *fileBackend) WriteAt(ctx context.Context, b []byte, offset int64, fua bool) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", len(b), offset)("n=%d, err=%v", &n, &err)
|
||||
n, err = fb.file.WriteAt(b, offset)
|
||||
if err != nil || !fua {
|
||||
return n, err
|
||||
}
|
||||
err = fb.file.Sync()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadAt implements Backend.ReadAt
|
||||
func (fb *fileBackend) ReadAt(ctx context.Context, b []byte, offset int64) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", len(b), offset)("n=%d, err=%v", &n, &err)
|
||||
return fb.file.ReadAt(b, offset)
|
||||
}
|
||||
|
||||
// TrimAt implements Backend.TrimAt
|
||||
func (fb *fileBackend) TrimAt(ctx context.Context, length int, offset int64) (n int, err error) {
|
||||
defer log.Trace(logPrefix, "size=%d, off=%d", length, offset)("n=%d, err=%v", &n, &err)
|
||||
return length, nil
|
||||
}
|
||||
|
||||
// Flush implements Backend.Flush
|
||||
func (fb *fileBackend) Flush(ctx context.Context) (err error) {
|
||||
defer log.Trace(logPrefix, "")("err=%v", &err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close implements Backend.Close
|
||||
func (fb *fileBackend) Close(ctx context.Context) (err error) {
|
||||
defer log.Trace(logPrefix, "")("err=%v", &err)
|
||||
err = fb.file.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Geometry implements Backend.Geometry
|
||||
func (fb *fileBackend) Geometry(ctx context.Context) (size uint64, minBS uint64, prefBS uint64, maxBS uint64, err error) {
|
||||
defer log.Trace(logPrefix, "")("size=%d, minBS=%d, prefBS=%d, maxBS=%d, err=%v", &size, &minBS, &prefBS, &maxBS, &err)
|
||||
fi, err := fb.file.Stat()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed read info about open backing file: %w", err)
|
||||
return
|
||||
}
|
||||
size = uint64(fi.Size())
|
||||
minBS = fb.ec.MinimumBlockSize
|
||||
prefBS = fb.ec.PreferredBlockSize
|
||||
maxBS = fb.ec.MaximumBlockSize
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
|
||||
// HasFua implements Backend.HasFua
|
||||
func (fb *fileBackend) HasFua(ctx context.Context) (fua bool) {
|
||||
defer log.Trace(logPrefix, "")("fua=%v", &fua)
|
||||
return true
|
||||
}
|
||||
|
||||
// HasFlush implements Backend.HasFua
|
||||
func (fb *fileBackend) HasFlush(ctx context.Context) (flush bool) {
|
||||
defer log.Trace(logPrefix, "")("flush=%v", &flush)
|
||||
return true
|
||||
}
|
||||
|
||||
// open the backing file
|
||||
func (fbf *fileBackendFactory) open() (vfs.Handle, error) {
|
||||
return fbf.vfs.OpenFile(fbf.filePath, fbf.perms, 0700)
|
||||
}
|
||||
|
||||
// New generates a new file backend
|
||||
func (fbf *fileBackendFactory) newBackend(ctx context.Context, ec *nbd.ExportConfig) (nbd.Backend, error) {
|
||||
fd, err := fbf.open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open backing file: %w", err)
|
||||
}
|
||||
fb := &fileBackend{
|
||||
file: fd,
|
||||
ec: ec,
|
||||
}
|
||||
return fb, nil
|
||||
}
|
||||
|
||||
// Generate a file backend factory
|
||||
func (s *NBD) newFileBackendFactory(ctx context.Context) (bf backendFactory, err error) {
|
||||
perms := os.O_RDWR
|
||||
if s.vfs.Opt.ReadOnly {
|
||||
perms = os.O_RDONLY
|
||||
}
|
||||
fbf := &fileBackendFactory{
|
||||
s: s,
|
||||
vfs: s.vfs,
|
||||
perms: perms,
|
||||
filePath: s.leaf,
|
||||
}
|
||||
// Try opening the file so we get errors now rather than later when they are more difficult to report.
|
||||
fd, err := fbf.open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open backing file: %w", err)
|
||||
}
|
||||
defer fs.CheckClose(fd, &err)
|
||||
return fbf, nil
|
||||
}
|
||||
|
||||
// Check interfaces
|
||||
var (
|
||||
_ nbd.Backend = (*fileBackend)(nil)
|
||||
_ backendFactory = (*fileBackendFactory)(nil)
|
||||
)
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
// Package nbd provides a network block device server
|
||||
package nbd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/bits"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/gonbdserver/nbd"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/lib/systemd"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/rclone/rclone/vfs/vfsflags"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
const logPrefix = "nbd"
|
||||
|
||||
// Options required for nbd server
|
||||
type Options struct {
|
||||
ListenAddr string // Port to listen on
|
||||
ReadOnly string // Set for read only
|
||||
MinBlockSize fs.SizeSuffix
|
||||
PreferredBlockSize fs.SizeSuffix
|
||||
MaxBlockSize fs.SizeSuffix
|
||||
Create fs.SizeSuffix
|
||||
ChunkSize fs.SizeSuffix
|
||||
Log2ChunkSize uint
|
||||
Resize fs.SizeSuffix
|
||||
// name := flag.String("name", "default", "Export name")
|
||||
// description := flag.String("description", "The default export", "Export description")
|
||||
}
|
||||
|
||||
// DefaultOpt is the default values used for Options
|
||||
var DefaultOpt = Options{
|
||||
ListenAddr: "localhost:10809",
|
||||
MinBlockSize: 512, // FIXME
|
||||
PreferredBlockSize: 4096, // this is the max according to nbd-client
|
||||
MaxBlockSize: 1024 * 1024, // FIXME
|
||||
}
|
||||
|
||||
// Opt is options set by command line flags
|
||||
var Opt = DefaultOpt
|
||||
|
||||
// AddFlags adds flags for the nbd
|
||||
func AddFlags(flagSet *pflag.FlagSet, Opt *Options) {
|
||||
rc.AddOption("nbd", &Opt)
|
||||
flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to", "")
|
||||
flags.FVarP(flagSet, &Opt.MinBlockSize, "min-block-size", "", "Minimum block size to advertise", "")
|
||||
flags.FVarP(flagSet, &Opt.PreferredBlockSize, "preferred-block-size", "", "Preferred block size to advertise", "")
|
||||
flags.FVarP(flagSet, &Opt.MaxBlockSize, "max-block-size", "", "Maximum block size to advertise", "")
|
||||
flags.FVarP(flagSet, &Opt.Create, "create", "", "If the destination does not exist, create it with this size", "")
|
||||
flags.FVarP(flagSet, &Opt.ChunkSize, "chunk-size", "", "If creating the destination use this chunk size. Must be a power of 2.", "")
|
||||
flags.FVarP(flagSet, &Opt.Resize, "resize", "", "If the destination does, resize it to this size", "")
|
||||
}
|
||||
|
||||
func init() {
|
||||
flagSet := Command.Flags()
|
||||
vfsflags.AddFlags(flagSet)
|
||||
proxyflags.AddFlags(flagSet)
|
||||
AddFlags(flagSet, &Opt)
|
||||
}
|
||||
|
||||
//go:embed nbd.md
|
||||
var helpText string
|
||||
|
||||
// Command definition for cobra
|
||||
var Command = &cobra.Command{
|
||||
Use: "nbd remote:path",
|
||||
Short: `Serve the remote over NBD.`,
|
||||
Long: helpText + vfs.Help,
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.65",
|
||||
"status": "experimental",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
// FIXME could serve more than one nbd?
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
f, leaf := cmd.NewFsFile(args[0])
|
||||
|
||||
cmd.Run(false, true, command, func() error {
|
||||
s, err := run(context.Background(), f, leaf, Opt)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
defer systemd.Notify()()
|
||||
// FIXME
|
||||
_ = s
|
||||
s.Wait()
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// NBD contains everything to run the server
|
||||
type NBD struct {
|
||||
f fs.Fs
|
||||
leaf string
|
||||
vfs *vfs.VFS // don't use directly, use getVFS
|
||||
opt Options
|
||||
wg sync.WaitGroup
|
||||
sessionWaitGroup sync.WaitGroup
|
||||
logRd *io.PipeReader
|
||||
logWr *io.PipeWriter
|
||||
|
||||
backendFactory backendFactory
|
||||
}
|
||||
|
||||
// interface for creating backend factories
|
||||
type backendFactory interface {
|
||||
newBackend(ctx context.Context, ec *nbd.ExportConfig) (nbd.Backend, error)
|
||||
}
|
||||
|
||||
// Create and start the server for nbd either on directory f or using file leaf in f
|
||||
func run(ctx context.Context, f fs.Fs, leaf string, opt Options) (s *NBD, err error) {
|
||||
if opt.ChunkSize != 0 {
|
||||
if set := bits.OnesCount64(uint64(opt.ChunkSize)); set != 1 {
|
||||
return nil, fmt.Errorf("--chunk-size must be a power of 2 (counted %d bits set)", set)
|
||||
}
|
||||
opt.Log2ChunkSize = uint(bits.TrailingZeros64(uint64(opt.ChunkSize)))
|
||||
fs.Debugf(logPrefix, "Using ChunkSize %v (%v), Log2ChunkSize %d", opt.ChunkSize, fs.SizeSuffix(1<<opt.Log2ChunkSize), opt.Log2ChunkSize)
|
||||
}
|
||||
if !vfsflags.Opt.ReadOnly && vfsflags.Opt.CacheMode < vfscommon.CacheModeWrites {
|
||||
return nil, errors.New("need --vfs-cache-mode writes or full when serving read/write")
|
||||
}
|
||||
|
||||
s = &NBD{
|
||||
f: f,
|
||||
leaf: leaf,
|
||||
opt: opt,
|
||||
vfs: vfs.New(f, &vfsflags.Opt),
|
||||
}
|
||||
|
||||
// Create the backend factory
|
||||
if leaf != "" {
|
||||
s.backendFactory, err = s.newFileBackendFactory(ctx)
|
||||
} else {
|
||||
s.backendFactory, err = s.newChunkedBackendFactory(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nbd.RegisterBackend("rclone", s.backendFactory.newBackend)
|
||||
fs.Debugf(logPrefix, "Registered backends: %v", nbd.GetBackendNames())
|
||||
|
||||
var (
|
||||
protocol = "tcp"
|
||||
addr = Opt.ListenAddr
|
||||
)
|
||||
if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) {
|
||||
protocol = "unix"
|
||||
addr = strings.TrimPrefix(addr, "unix://")
|
||||
|
||||
}
|
||||
|
||||
ec := nbd.ExportConfig{
|
||||
Name: "default",
|
||||
Description: fs.ConfigString(f),
|
||||
Driver: "rclone",
|
||||
ReadOnly: vfsflags.Opt.ReadOnly,
|
||||
Workers: 8, // should this be --checkers or a new config flag FIXME
|
||||
TLSOnly: false, // FIXME
|
||||
MinimumBlockSize: uint64(Opt.MinBlockSize),
|
||||
PreferredBlockSize: uint64(Opt.PreferredBlockSize),
|
||||
MaximumBlockSize: uint64(Opt.MaxBlockSize),
|
||||
DriverParameters: nbd.DriverParametersConfig{
|
||||
"sync": "false",
|
||||
"path": "/tmp/diskimage",
|
||||
},
|
||||
}
|
||||
|
||||
// Make a logger to feed gonbdserver's logs into rclone's logging system
|
||||
s.logRd, s.logWr = io.Pipe()
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(s.logRd)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if s, ok := strings.CutPrefix(line, "[DEBUG] "); ok {
|
||||
fs.Debugf(logPrefix, "%s", s)
|
||||
} else if s, ok := strings.CutPrefix(line, "[INFO] "); ok {
|
||||
fs.Infof(logPrefix, "%s", s)
|
||||
} else if s, ok := strings.CutPrefix(line, "[WARN] "); ok {
|
||||
fs.Logf(logPrefix, "%s", s)
|
||||
} else if s, ok := strings.CutPrefix(line, "[ERROR] "); ok {
|
||||
fs.Errorf(logPrefix, "%s", s)
|
||||
} else if s, ok := strings.CutPrefix(line, "[CRIT] "); ok {
|
||||
fs.Errorf(logPrefix, "%s", s)
|
||||
} else {
|
||||
fs.Infof(logPrefix, "%s", line)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
fs.Errorf(logPrefix, "Log writer failed: %v", err)
|
||||
}
|
||||
}()
|
||||
logger := log.New(s.logWr, "", 0)
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
dump := ci.Dump & (fs.DumpHeaders | fs.DumpBodies | fs.DumpAuth | fs.DumpRequests | fs.DumpResponses)
|
||||
var serverConfig = nbd.ServerConfig{
|
||||
Protocol: protocol, // protocol it should listen on (in net.Conn form)
|
||||
Address: addr, // address to listen on
|
||||
DefaultExport: "default", // name of default export
|
||||
Exports: []nbd.ExportConfig{ec}, // array of configurations of exported items
|
||||
//TLS: nbd.TLSConfig{}, // TLS configuration
|
||||
DisableNoZeroes: false, // Disable NoZereos extension FIXME
|
||||
Debug: dump != 0, // Verbose debug
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
// FIXME contexts
|
||||
nbd.StartServer(ctx, ctx, &s.sessionWaitGroup, logger, serverConfig)
|
||||
}()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Wait for the server to finish
|
||||
func (s *NBD) Wait() {
|
||||
s.wg.Wait()
|
||||
_ = s.logWr.Close()
|
||||
_ = s.logRd.Close()
|
||||
}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
Run a Network Block Device server using remote:path to store the object.
|
||||
|
||||
You can use a unix socket by setting the url to `unix:/path/to/socket`
|
||||
or just by using an absolute path name.
|
||||
|
||||
`rclone serve nbd` will run on any OS, but the examples for using it
|
||||
are Linux specific. There do exist Windows and macOS NBD clients but
|
||||
these haven't been tested yet.
|
||||
|
||||
To see the packets on the wire use `--dump headers` or `--dump bodies`.
|
||||
|
||||
**NB** this has no authentication. It may in the future allow SSL
|
||||
certificates. If you need access control then you will have to provide
|
||||
it on the network layer, or use unix sockets.
|
||||
|
||||
### remote:path pointing to a file
|
||||
|
||||
If the `remote:path` points to a file then rclone will serve the file
|
||||
directly as a network block device.
|
||||
|
||||
Using this with `--read-only` is recommended. You can use any
|
||||
`--vfs-cache-mode` and only parts of the file that are read will be
|
||||
cached locally if using `--vfs-cache-mode full`.
|
||||
|
||||
If you don't use `--read-only` then `--vfs-cache-mode full` is
|
||||
required and the entire file will be cached locally and won't be
|
||||
uploaded until the client has disconnected (`nbd-client -d`).
|
||||
|
||||
### remote:path pointing to a directory
|
||||
|
||||
If the `remote:path` points to a directory then rclone will treat the
|
||||
directory as a place to store chunks of the exported network block device.
|
||||
|
||||
It will store an `info.json` file in the top level and store the
|
||||
individual chunks in a hierarchical directory scheme with no more than
|
||||
256 chunks or directories in any directory.
|
||||
|
||||
The first time you use this, you should use the `--create` flag
|
||||
indicating how big you want the network block device to appear. Rclone
|
||||
only allocates space you use so you can make this large. For example
|
||||
`--create 1T`. You can also pass the `--chunk-size` flag at this
|
||||
point. If you don't you will get the default of 64k chunks.
|
||||
|
||||
Rclone will then chunk the network block device into `--chunk-size`
|
||||
chunks. Rclone has to download the entire chunk in order to change
|
||||
only part of it and it will cache the chunk on disk so bear that in
|
||||
mind when choosing `--chunk-size`.
|
||||
|
||||
If you wish to change the size of the network block device you can use
|
||||
the `--resize` flag. This won't remove any data, it just changes the
|
||||
size advertised. So if you have made a file system on the block device
|
||||
you will need to resize it too.
|
||||
|
||||
If you are using `--read-only` then you can use any
|
||||
`--vfs-cache-mode`.
|
||||
|
||||
If you are not using `--read-only` then you will need
|
||||
`--vfs-cache-mode writes` or `--vfs-cache-mode full`.
|
||||
|
||||
Note that rclone will be acting as a writeback cache with
|
||||
`--vfs-cache-mode writes` or `--vfs-cache-mode full`. Note that rclone
|
||||
will only write `--transfers` files at once so the cache can get a
|
||||
backlog of uploads. You can reduce the writeback caching slightly
|
||||
setting `--vfs-write-back 0`, however due to the way the kernel works,
|
||||
this will only reduce it slightly.
|
||||
|
||||
If using `--vfs-cache-mode writes` or `--vfs-cache-mode full` it is
|
||||
recommended to set limits on the cache size using some or all of these
|
||||
flags as the VFS can use a lot of disk space very quickly.
|
||||
|
||||
--vfs-cache-max-age duration Max time since last access of objects in the cache (default 1h0m0s)
|
||||
--vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off)
|
||||
--vfs-cache-min-free-space SizeSuffix Target minimum free space on the disk containing the cache (default off)
|
||||
|
||||
You might also need to set this smaller as the cache will only be
|
||||
examined at this interval.
|
||||
|
||||
--vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s)
|
||||
|
||||
### Linux Examples
|
||||
|
||||
Install
|
||||
|
||||
sudo apt install nbd-client
|
||||
|
||||
Start server on localhost:10809 by default.
|
||||
|
||||
rclone -v --vfs-cache-mode full serve ndb remote:path
|
||||
|
||||
List devices
|
||||
|
||||
sudo modprobe nbd
|
||||
sudo nbd-client --list localhost
|
||||
|
||||
Format the partition and mount read write
|
||||
|
||||
sudo nbd-client -g localhost 10809 /dev/nbd0
|
||||
sudo mkfs.ext4 /dev/nbd0
|
||||
sudo mkdir -p /mnt/tmp
|
||||
sudo mount -t ext4 /dev/nbd0 /mnt/tmp
|
||||
|
||||
Mount read only
|
||||
|
||||
rclone -v --vfs-cache-mode full --read-only serve ndb remote:path
|
||||
sudo nbd-client --readonly -g localhost 10809 /dev/nbd0
|
||||
sudo mount -t ext4 -o ro /dev/nbd0 /mnt/tmp
|
||||
|
||||
Disconnect
|
||||
|
||||
sudo umount /mnt/tmp
|
||||
sudo nbd-client -d /dev/nbd0
|
||||
|
||||
### TODO
|
||||
|
||||
Experiment with `-connections` option. This is supported by the code.
|
||||
Does it improve performance?
|
||||
|
||||
-connections num
|
||||
|
||||
-C Use num connections to the server, to allow speeding up request
|
||||
handling, at the cost of higher resource usage on the server.
|
||||
Use of this option requires kernel support available first with
|
||||
Linux 4.9.
|
||||
|
||||
Experiment with `-persist` option - is that a good idea?
|
||||
|
||||
-persist
|
||||
|
||||
-p When this option is specified, nbd-client will immediately try
|
||||
to reconnect an nbd device if the connection ever drops unex‐
|
||||
pectedly due to a lost server or something similar.
|
||||
|
||||
Need to implement Trim and see if Trim is being called.
|
||||
|
||||
Need to delete zero files before upload (do in VFS layer?)
|
||||
|
||||
FIXME need better back pressure from VFS cache to writers.
|
||||
|
||||
FIXME need Sync to actually work!
|
||||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/rclone/rclone/cmd/serve/docker"
|
||||
"github.com/rclone/rclone/cmd/serve/ftp"
|
||||
"github.com/rclone/rclone/cmd/serve/http"
|
||||
"github.com/rclone/rclone/cmd/serve/nbd"
|
||||
"github.com/rclone/rclone/cmd/serve/restic"
|
||||
"github.com/rclone/rclone/cmd/serve/sftp"
|
||||
"github.com/rclone/rclone/cmd/serve/webdav"
|
||||
|
|
@ -35,6 +36,9 @@ func init() {
|
|||
if docker.Command != nil {
|
||||
Command.AddCommand(docker.Command)
|
||||
}
|
||||
if nbd.Command != nil {
|
||||
Command.AddCommand(nbd.Command)
|
||||
}
|
||||
cmd.Root.AddCommand(Command)
|
||||
}
|
||||
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -2,6 +2,8 @@ module github.com/rclone/rclone
|
|||
|
||||
go 1.19
|
||||
|
||||
//replace github.com/rclone/gonbdserver => /home/ncw/go/src/github.com/rclone/gonbdserver
|
||||
|
||||
require (
|
||||
bazil.org/fuse v0.0.0-20221209211307-2abb8038c751
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
|
||||
|
|
@ -48,6 +50,7 @@ require (
|
|||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6
|
||||
github.com/pmezard/go-difflib v1.0.0
|
||||
github.com/pojntfx/go-nbd v0.2.0
|
||||
github.com/prometheus/client_golang v1.15.1
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
|
||||
github.com/rfjakob/eme v1.1.2
|
||||
|
|
@ -124,6 +127,7 @@ require (
|
|||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
|
|
@ -132,14 +136,17 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/pilebones/go-udev v0.9.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0 // indirect
|
||||
github.com/rclone/gonbdserver v0.0.0-20230928185136-7adb4642e1cb // indirect
|
||||
github.com/relvacode/iso8601 v1.3.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sevlyar/go-daemon v0.1.6 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -341,6 +341,8 @@ github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 h1:+A1uT26XjTsxiU
|
|||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8/go.mod h1:f0ijQHcvHYAuxX6JA/JUr/Z0FVn12D9REaT/HAWVgP4=
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg=
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
|
||||
|
|
@ -408,6 +410,8 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ
|
|||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pilebones/go-udev v0.9.0 h1:N1uEO/SxUwtIctc0WLU0t69JeBxIYEYnj8lT/Nabl9Q=
|
||||
github.com/pilebones/go-udev v0.9.0/go.mod h1:T2eI2tUSK0hA2WS5QLjXJUfQkluZQu+18Cqvem3CaXI=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
|
|
@ -419,6 +423,8 @@ github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
|||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pojntfx/go-nbd v0.2.0 h1:dBtrQQQ2b4Gr3zNb1Lxzfs1/hnEVtHGJeUEC6GVsBO4=
|
||||
github.com/pojntfx/go-nbd v0.2.0/go.mod h1:SehHnbi2e8NiSAKby42Itm8SIoS7b+wAprsfPH3qgYk=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
|
||||
|
|
@ -434,6 +440,10 @@ github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzX
|
|||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
|
||||
github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
|
||||
github.com/rclone/gonbdserver v0.0.0-20230928151619-3fcbe3a09b04 h1:98xAAGVyZtB097H+pL4BjkWzHUmlHh4fhGtkGZQXvMY=
|
||||
github.com/rclone/gonbdserver v0.0.0-20230928151619-3fcbe3a09b04/go.mod h1:HwROhGq4gA7vncM5mLZjoNbI9CrS52aDuHReB3NMWp4=
|
||||
github.com/rclone/gonbdserver v0.0.0-20230928185136-7adb4642e1cb h1:4FyF15nQLPIhLcJDpn2ItwcuO3E/pYQXdPVOt+v3Duk=
|
||||
github.com/rclone/gonbdserver v0.0.0-20230928185136-7adb4642e1cb/go.mod h1:HwROhGq4gA7vncM5mLZjoNbI9CrS52aDuHReB3NMWp4=
|
||||
github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko=
|
||||
github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
|
|
@ -450,6 +460,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
|
|||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/sevlyar/go-daemon v0.1.6 h1:EUh1MDjEM4BI109Jign0EaknA2izkOyi0LV3ro3QQGs=
|
||||
github.com/sevlyar/go-daemon v0.1.6/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE=
|
||||
github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08=
|
||||
github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
|
|
|
|||
Loading…
Reference in New Issue