mirror of https://github.com/rclone/rclone
archive backend - WIP FIXME
- Tests all passing. - Zip and squashfs archivers working. - Lazily loads and caches squashfs Squashfs - see archive.go for more FIXMEs
This commit is contained in:
Nick Craig-Wood
parent
65d1e5d47e
commit
305183e6c5
|
|
@ -0,0 +1,677 @@
|
|||
// Package archive implements a backend to archive multiple remotes in a directory tree
|
||||
package archive
|
||||
|
||||
// FIXME factor common code between backends out - eg VFS initialization
|
||||
|
||||
// FIXME can we generalize the VFS handle caching and use it in zip backend
|
||||
|
||||
// Factor more stuff out if possible
|
||||
|
||||
// Odd stats which are probably coming from the VFS
|
||||
// * tensorflow.sqfs: 0% /3.074Gi, 204.426Ki/s, 4h22m46s
|
||||
|
||||
// FIXME this will perform poorly for unpacking as the VFS Reader is bad
|
||||
// at multiple streams - need cache mode setting?
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// Import all the required archivers here
|
||||
_ "github.com/rclone/rclone/backend/archive/squashfs"
|
||||
_ "github.com/rclone/rclone/backend/archive/zip"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fsi := &fs.RegInfo{
|
||||
Name: "archive",
|
||||
Description: "Read archives",
|
||||
NewFs: NewFs,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
Help: `Any metadata supported by the underlying remote is read and written.`,
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "remote",
|
||||
Help: `Remote to wrap to read archives from.
|
||||
|
||||
Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
|
||||
"myremote:bucket" or "myremote:".
|
||||
|
||||
If this is left empty, then the archive backend will use the root as
|
||||
the remote.
|
||||
|
||||
This means that you can use :archive:remote:path and it will be
|
||||
equivalent to setting remote="remote:path".
|
||||
`,
|
||||
Required: false,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
}
|
||||
|
||||
// Fs represents a archive of upstreams
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
features *fs.Features // optional features
|
||||
opt Options // options for this Fs
|
||||
root string // the path we are working on
|
||||
f fs.Fs // remote we are wrapping
|
||||
wrapper fs.Fs // fs that wraps us
|
||||
|
||||
mu sync.Mutex // protects the below
|
||||
archives map[string]*archive // the archives we have, by path
|
||||
}
|
||||
|
||||
// A single open archive
|
||||
type archive struct {
|
||||
archiver archiver.Archiver // archiver responsible
|
||||
remote string // path to the archive
|
||||
prefix string // prefix to add on to listings
|
||||
root string // root of the archive to remove from listings
|
||||
mu sync.Mutex // protects the following variables
|
||||
f fs.Fs // the archive Fs, may be nil
|
||||
}
|
||||
|
||||
// If remote is an archive then return it otherwise return nil
|
||||
func findArchive(remote string) *archive {
|
||||
// FIXME use something faster than linear search?
|
||||
for _, archiver := range archiver.Archivers {
|
||||
if strings.HasSuffix(remote, archiver.Extension) {
|
||||
return &archive{
|
||||
archiver: archiver,
|
||||
remote: remote,
|
||||
prefix: remote,
|
||||
root: "",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find an archive buried in remote
|
||||
func subArchive(remote string) *archive {
|
||||
archive := findArchive(remote)
|
||||
if archive != nil {
|
||||
return archive
|
||||
}
|
||||
parent := path.Dir(remote)
|
||||
if parent == "/" || parent == "." {
|
||||
return nil
|
||||
}
|
||||
return subArchive(parent)
|
||||
}
|
||||
|
||||
// If remote is an archive then return it otherwise return nil
|
||||
func (f *Fs) findArchive(remote string) (archive *archive) {
|
||||
archive = findArchive(remote)
|
||||
if archive != nil {
|
||||
f.mu.Lock()
|
||||
f.archives[remote] = archive
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return archive
|
||||
}
|
||||
|
||||
// Instantiate archive if it hasn't been instantiated yet
|
||||
//
|
||||
// This is done lazily so that we can list a directory full of
|
||||
// archives without opening them all.
|
||||
func (a *archive) init(ctx context.Context, f fs.Fs) (fs.Fs, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.f != nil {
|
||||
return a.f, nil
|
||||
}
|
||||
newFs, err := a.archiver.New(ctx, f, a.remote, a.prefix, a.root)
|
||||
if err != nil && err != fs.ErrorIsFile {
|
||||
return nil, fmt.Errorf("failed to create archive %q: %w", a.remote, err)
|
||||
}
|
||||
a.f = newFs
|
||||
return a.f, nil
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path.
|
||||
//
|
||||
// The returned Fs is the actual Fs, referenced by remote in the config
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
|
||||
// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err = configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := opt.Remote
|
||||
origRoot := root
|
||||
|
||||
// If remote is empty, use the root instead
|
||||
if remote == "" {
|
||||
remote = root
|
||||
root = ""
|
||||
}
|
||||
isDirectory := strings.HasSuffix(remote, "/")
|
||||
remote = strings.TrimRight(remote, "/")
|
||||
if remote == "" {
|
||||
remote = "/"
|
||||
}
|
||||
if strings.HasPrefix(remote, name+":") {
|
||||
return nil, errors.New("can't point archive remote at itself - check the value of the upstreams setting")
|
||||
}
|
||||
|
||||
_ = isDirectory
|
||||
|
||||
foundArchive := subArchive(remote)
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
|
||||
// Archive path
|
||||
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name
|
||||
foundArchive.remote = path.Base(archiveRemote)
|
||||
foundArchive.prefix = ""
|
||||
// Point remote to archive file
|
||||
remote = archiveRemote
|
||||
}
|
||||
|
||||
// Make sure to remove trailing . referring to the current dir
|
||||
if path.Base(root) == "." {
|
||||
root = strings.TrimSuffix(root, ".")
|
||||
}
|
||||
remotePath := fspath.JoinRootPath(remote, root)
|
||||
wrappedFs, err := cache.Get(ctx, remotePath)
|
||||
if err != fs.ErrorIsFile && err != nil {
|
||||
return nil, fmt.Errorf("failed to make remote %q to wrap: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
//root: path.Join(remotePath, root),
|
||||
root: origRoot,
|
||||
opt: *opt,
|
||||
f: wrappedFs,
|
||||
archives: make(map[string]*archive),
|
||||
}
|
||||
cache.PinUntilFinalized(f.f, f)
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
BucketBased: true,
|
||||
SetTier: true,
|
||||
GetTier: true,
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: true,
|
||||
PartialUploads: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(f, "Root is an archive")
|
||||
if err != fs.ErrorIsFile {
|
||||
return nil, fmt.Errorf("expecting to find a file at %q", remote)
|
||||
}
|
||||
return foundArchive.init(ctx, f.f)
|
||||
}
|
||||
// Correct root if definitely pointing to a file
|
||||
if err == fs.ErrorIsFile {
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("archive root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// Rmdir removes the root directory of the Fs object
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.f.Rmdir(ctx, dir)
|
||||
}
|
||||
|
||||
// Hashes returns hash.HashNone to indicate remote hashing is unavailable
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return f.f.Hashes()
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return f.f.Mkdir(ctx, dir)
|
||||
}
|
||||
|
||||
// Purge all files in the directory
|
||||
//
|
||||
// Implement this if you have a way of deleting all the files
|
||||
// quicker than just running Remove() on the result of List()
|
||||
//
|
||||
// Return an error if it doesn't exist
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
do := f.f.Features().Purge
|
||||
if do == nil {
|
||||
return fs.ErrorCantPurge
|
||||
}
|
||||
return do(ctx, dir)
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantCopy
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
do := f.f.Features().Copy
|
||||
if do == nil {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
// FIXME
|
||||
// o, ok := src.(*Object)
|
||||
// if !ok {
|
||||
// return nil, fs.ErrorCantCopy
|
||||
// }
|
||||
return do(ctx, src, remote)
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantMove
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
do := f.f.Features().Move
|
||||
if do == nil {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
// FIXME
|
||||
// o, ok := src.(*Object)
|
||||
// if !ok {
|
||||
// return nil, fs.ErrorCantMove
|
||||
// }
|
||||
return do(ctx, src, remote)
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantDirMove
|
||||
//
|
||||
// If destination exists then return fs.ErrorDirExists
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
|
||||
do := f.f.Features().DirMove
|
||||
if do == nil {
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
return do(ctx, srcFs.f, srcRemote, dstRemote)
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path
|
||||
// that has had changes. If the implementation
|
||||
// uses polling, it should adhere to the given interval.
|
||||
// At least one value will be written to the channel,
|
||||
// specifying the initial value and updated values might
|
||||
// follow. A 0 Duration should pause the polling.
|
||||
// The ChangeNotify implementation must empty the channel
|
||||
// regularly. When the channel gets closed, the implementation
|
||||
// should stop polling and release resources.
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) {
|
||||
do := f.f.Features().ChangeNotify
|
||||
if do == nil {
|
||||
return
|
||||
}
|
||||
wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
|
||||
// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
|
||||
notifyFunc(path, entryType)
|
||||
}
|
||||
do(ctx, wrappedNotifyFunc, ch)
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing
|
||||
// as an optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
do := f.f.Features().DirCacheFlush
|
||||
if do != nil {
|
||||
do()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
|
||||
var o fs.Object
|
||||
var err error
|
||||
if stream {
|
||||
o, err = f.f.Features().PutStream(ctx, in, src, options...)
|
||||
} else {
|
||||
o, err = f.f.Put(ctx, in, src, options...)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, false, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, true, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.f.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Find the Fs for the directory
|
||||
func (f *Fs) findFs(ctx context.Context, dir string) (subFs fs.Fs, err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
subFs = f.f
|
||||
|
||||
// FIXME should do this with a better datastructure like a prefix tree
|
||||
// FIXME want to find the longest first otherwise nesting won't work
|
||||
dirSlash := dir + "/"
|
||||
for archiverRemote, archive := range f.archives {
|
||||
subRemote := archiverRemote + "/"
|
||||
if strings.HasPrefix(dirSlash, subRemote) {
|
||||
subFs, err = archive.init(ctx, f.f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return subFs, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
|
||||
|
||||
subFs, err := f.findFs(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries, err = subFs.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, entry := range entries {
|
||||
// Can only unarchive files
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
remote := o.Remote()
|
||||
archive := f.findArchive(remote)
|
||||
if archive != nil {
|
||||
// Overwrite entry with directory
|
||||
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
|
||||
}
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
|
||||
subFs, err := f.findFs(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
o, err := subFs.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Precision is the greatest precision of all the archivers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Shutdown the backend, closing any background tasks and any
|
||||
// cached connections.
|
||||
func (f *Fs) Shutdown(ctx context.Context) error {
|
||||
if do := f.f.Features().Shutdown; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
do := f.f.Features().PublicLink
|
||||
if do == nil {
|
||||
return "", errors.New("PublicLink not supported")
|
||||
}
|
||||
return do(ctx, remote, expire, unlink)
|
||||
}
|
||||
|
||||
// PutUnchecked in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
//
|
||||
// May create duplicates or return errors if src already
|
||||
// exists.
|
||||
func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
do := f.f.Features().PutUnchecked
|
||||
if do == nil {
|
||||
return nil, errors.New("can't PutUnchecked")
|
||||
}
|
||||
o, err := do(ctx, in, src, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// MergeDirs merges the contents of all the directories passed
|
||||
// in into the first one and rmdirs the other directories.
|
||||
func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
if len(dirs) == 0 {
|
||||
return nil
|
||||
}
|
||||
do := f.f.Features().MergeDirs
|
||||
if do == nil {
|
||||
return errors.New("MergeDirs not supported")
|
||||
}
|
||||
return do(ctx, dirs)
|
||||
}
|
||||
|
||||
// CleanUp the trash in the Fs
|
||||
//
|
||||
// Implement this if you have a way of emptying the trash or
|
||||
// otherwise cleaning up old versions of files.
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.f.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// OpenWriterAt opens with a handle for random access writes
|
||||
//
|
||||
// Pass in the remote desired and the size if known.
|
||||
//
|
||||
// It truncates any existing object
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
|
||||
do := f.f.Features().OpenWriterAt
|
||||
if do == nil {
|
||||
return nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx, remote, size)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
||||
//
|
||||
// Pass in the remote and the src object
|
||||
// You can also use options to hint at the desired chunk size
|
||||
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
|
||||
do := f.f.Features().OpenChunkWriter
|
||||
if do == nil {
|
||||
return info, nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx, remote, src, options...)
|
||||
}
|
||||
|
||||
// UserInfo returns info about the connected user
|
||||
func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
|
||||
do := f.f.Features().UserInfo
|
||||
if do == nil {
|
||||
return nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Disconnect the current user
|
||||
func (f *Fs) Disconnect(ctx context.Context) error {
|
||||
do := f.f.Features().Disconnect
|
||||
if do == nil {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PutStreamer = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.ChangeNotifier = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.Shutdowner = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.PutUncheckeder = (*Fs)(nil)
|
||||
_ fs.MergeDirser = (*Fs)(nil)
|
||||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.OpenWriterAter = (*Fs)(nil)
|
||||
_ fs.OpenChunkWriter = (*Fs)(nil)
|
||||
_ fs.UserInfoer = (*Fs)(nil)
|
||||
_ fs.Disconnecter = (*Fs)(nil)
|
||||
// FIXME _ fs.FullObject = (*Object)(nil)
|
||||
)
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// FIXME need to test Open with seek
|
||||
// FIXME need to test NewObject
|
||||
|
||||
// run - run a shell command
|
||||
func run(t *testing.T, args ...string) {
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
fs.Debugf(nil, "run args = %v", args)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf(`
|
||||
----------------------------
|
||||
Failed to run %v: %v
|
||||
Command output was:
|
||||
%s
|
||||
----------------------------
|
||||
`, args, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
// check the dst and src are identical
|
||||
func checkTree(ctx context.Context, name string, t *testing.T, dstArchive, src string, expectedCount int) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fs.Debugf(nil, "check %q vs %q", dstArchive, src)
|
||||
Farchive, err := cache.Get(ctx, dstArchive)
|
||||
if err != fs.ErrorIsFile {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
Fsrc, err := cache.Get(ctx, src)
|
||||
if err != fs.ErrorIsFile {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var matches bytes.Buffer
|
||||
opt := operations.CheckOpt{
|
||||
Fdst: Farchive,
|
||||
Fsrc: Fsrc,
|
||||
Match: &matches,
|
||||
}
|
||||
|
||||
for _, action := range []string{"Check", "Download"} {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
matches.Reset()
|
||||
if action == "Download" {
|
||||
assert.NoError(t, operations.CheckDownload(ctx, &opt))
|
||||
} else {
|
||||
assert.NoError(t, operations.Check(ctx, &opt))
|
||||
}
|
||||
if expectedCount > 0 {
|
||||
assert.Equal(t, expectedCount, strings.Count(matches.String(), "\n"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("NewObject", func(t *testing.T) {
|
||||
// Check we can run NewObject on all files and read them
|
||||
assert.NoError(t, operations.ListFn(ctx, Fsrc, func(srcObj fs.Object) {
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
remote := srcObj.Remote()
|
||||
archiveObj, err := Farchive.NewObject(ctx, remote)
|
||||
require.NoError(t, err, remote)
|
||||
assert.Equal(t, remote, archiveObj.Remote(), remote)
|
||||
|
||||
// Test that the contents are the same
|
||||
archiveBuf := fstests.ReadObject(ctx, t, archiveObj, -1)
|
||||
srcBuf := fstests.ReadObject(ctx, t, srcObj, -1)
|
||||
assert.Equal(t, srcBuf, archiveBuf)
|
||||
|
||||
if len(srcBuf) < 81 {
|
||||
return
|
||||
}
|
||||
|
||||
// Tests that Open works with SeekOption
|
||||
assert.Equal(t, srcBuf[50:], fstests.ReadObject(ctx, t, archiveObj, -1, &fs.SeekOption{Offset: 50}), "contents differ after seek")
|
||||
|
||||
// Tests that Open works with RangeOption
|
||||
for _, test := range []struct {
|
||||
ro fs.RangeOption
|
||||
wantStart, wantEnd int
|
||||
}{
|
||||
{fs.RangeOption{Start: 5, End: 15}, 5, 16},
|
||||
{fs.RangeOption{Start: 80, End: -1}, 80, len(srcBuf)},
|
||||
{fs.RangeOption{Start: 81, End: 100000}, 81, len(srcBuf)},
|
||||
{fs.RangeOption{Start: -1, End: 20}, len(srcBuf) - 20, len(srcBuf)}, // if start is omitted this means get the final bytes
|
||||
// {fs.RangeOption{Start: -1, End: -1}, 0, len(srcBuf)}, - this seems to work but the RFC doesn't define it
|
||||
} {
|
||||
got := fstests.ReadObject(ctx, t, archiveObj, -1, &test.ro)
|
||||
foundAt := strings.Index(srcBuf, got)
|
||||
help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got))
|
||||
assert.Equal(t, srcBuf[test.wantStart:test.wantEnd], got, help)
|
||||
}
|
||||
|
||||
// Test that the modtimes are correct
|
||||
fstest.AssertTimeEqualWithPrecision(t, remote, srcObj.ModTime(ctx), archiveObj.ModTime(ctx), Farchive.Precision())
|
||||
|
||||
// Test that the sizes are correct
|
||||
assert.Equal(t, srcObj.Size(), archiveObj.Size())
|
||||
|
||||
// Test that Strings are OK
|
||||
assert.Equal(t, srcObj.String(), archiveObj.String())
|
||||
}))
|
||||
})
|
||||
|
||||
// t.Logf("Fdst ------------- %v", Fdst)
|
||||
// operations.List(ctx, Fdst, os.Stdout)
|
||||
// t.Logf("Fsrc ------------- %v", Fsrc)
|
||||
// operations.List(ctx, Fsrc, os.Stdout)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and zip as external binaries.
|
||||
func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T, output, input string)) {
|
||||
ctx := context.Background()
|
||||
checkFiles := 1000
|
||||
|
||||
// create random test input files
|
||||
inputRoot := t.TempDir()
|
||||
input := filepath.Join(inputRoot, archiveName)
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
run(t, "rclone", "test", "makefiles", "--files", strconv.Itoa(checkFiles), "--ascii", input)
|
||||
|
||||
// Create the archive
|
||||
output := t.TempDir()
|
||||
zipFile := path.Join(output, archiveName)
|
||||
archiveFn(t, zipFile, input)
|
||||
|
||||
// Check the archive itself
|
||||
checkTree(ctx, "Archive", t, ":archive:"+zipFile, input, checkFiles)
|
||||
|
||||
// Now check a subdirectory
|
||||
fis, err := os.ReadDir(input)
|
||||
require.NoError(t, err)
|
||||
subDir := "NOT FOUND"
|
||||
aFile := "NOT FOUND"
|
||||
for _, fi := range fis {
|
||||
if fi.IsDir() {
|
||||
subDir = fi.Name()
|
||||
} else {
|
||||
aFile = fi.Name()
|
||||
}
|
||||
}
|
||||
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
|
||||
|
||||
// Now check a single file
|
||||
fiCtx, fi := filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("+ "+aFile))
|
||||
require.NoError(t, fi.AddRule("- *"))
|
||||
checkTree(fiCtx, "SingleFile", t, ":archive:"+zipFile+"/"+aFile, filepath.Join(input, aFile), 0)
|
||||
|
||||
// Now check the level above
|
||||
checkTree(ctx, "Root", t, ":archive:"+output, inputRoot, checkFiles)
|
||||
// run(t, "cp", "-a", inputRoot, output, "/tmp/test-"+archiveName)
|
||||
}
|
||||
|
||||
// Make sure we have the executable named
|
||||
func skipIfNoExe(t *testing.T, exeName string) {
|
||||
_, err := exec.LookPath(exeName)
|
||||
if err != nil {
|
||||
t.Skipf("%s executable not installed", exeName)
|
||||
}
|
||||
}
|
||||
|
||||
// Test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and zip as external binaries.
|
||||
func TestArchiveZip(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "zip")
|
||||
skipIfNoExe(t, "rclone")
|
||||
testArchive(t, "test.zip", func(t *testing.T, output, input string) {
|
||||
oldcwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(input))
|
||||
defer func() {
|
||||
require.NoError(t, os.Chdir(oldcwd))
|
||||
}()
|
||||
run(t, "zip", "-9r", output, ".")
|
||||
})
|
||||
}
|
||||
|
||||
// Test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and squashfs as external binaries.
|
||||
func TestArchiveSquashfs(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
skipIfNoExe(t, "rclone")
|
||||
testArchive(t, "test.sqfs", func(t *testing.T, output, input string) {
|
||||
run(t, "mksquashfs", input, output)
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
// Test Archive filesystem interface
|
||||
package archive_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
var (
|
||||
unimplementableFsMethods = []string{"ListR"}
|
||||
// In these tests we receive objects from the underlying remote which don't implement these methods
|
||||
unimplementableObjectMethods = []string{"GetTier", "ID", "Metadata", "MimeType", "SetTier", "UnWrap"}
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
if *fstest.RemoteName == "" {
|
||||
t.Skip("Skipping as -remote not set")
|
||||
}
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: *fstest.RemoteName,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocal(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
remote := t.TempDir()
|
||||
name := "TestArchiveLocal"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "archive"},
|
||||
{Name: name, Key: "remote", Value: remote},
|
||||
},
|
||||
QuickTestOK: true,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
|
||||
func TestMemory(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
remote := ":memory:"
|
||||
name := "TestArchiveMemory"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "archive"},
|
||||
{Name: name, Key: "remote", Value: remote},
|
||||
},
|
||||
QuickTestOK: true,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
// Package archiver registers all the archivers
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
// Archiver describes an archive package
|
||||
type Archiver struct {
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
New func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error)
|
||||
Extension string
|
||||
}
|
||||
|
||||
// Archivers is a slice of all registered archivers
|
||||
var Archivers []Archiver
|
||||
|
||||
// Register adds the archivers provided to the list of known archivers
|
||||
func Register(as ...Archiver) {
|
||||
Archivers = append(Archivers, as...)
|
||||
}
|
||||
|
|
@ -0,0 +1,233 @@
|
|||
// Package base is a base archive Fs
|
||||
package base
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
node vfs.Node // archive object
|
||||
remote string // remote of the archive object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
}
|
||||
|
||||
var errNotImplemented = errors.New("internal error: method not implemented in archiver")
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
VFS := vfs.New(wrappedFs, nil)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gzip
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw zip file
|
||||
type Object struct {
|
||||
f *Fs
|
||||
remote string
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.f
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return -1
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package squashfs
|
||||
|
||||
// Could just be using bare object Open with RangeRequest which
|
||||
// would transfer the minimum amount of data but may be slower.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
// Cache file handles for accessing the file
|
||||
type cache struct {
|
||||
node vfs.Node
|
||||
fhsMu sync.Mutex
|
||||
fhs []cacheHandle
|
||||
}
|
||||
|
||||
// A cached file handle
|
||||
type cacheHandle struct {
|
||||
offset int64
|
||||
fh vfs.Handle
|
||||
}
|
||||
|
||||
// Make a new cache
|
||||
func newCache(node vfs.Node) *cache {
|
||||
return &cache{
|
||||
node: node,
|
||||
}
|
||||
}
|
||||
|
||||
// Get a vfs.Handle from the pool or open one
|
||||
//
|
||||
// This tries to find an open file handle which doesn't require seeking.
|
||||
func (c *cache) open(off int64) (fh vfs.Handle, err error) {
|
||||
c.fhsMu.Lock()
|
||||
defer c.fhsMu.Unlock()
|
||||
|
||||
if len(c.fhs) > 0 {
|
||||
// Look for exact match first
|
||||
for i, cfh := range c.fhs {
|
||||
if cfh.offset == off {
|
||||
// fs.Debugf(nil, "CACHE MATCH")
|
||||
c.fhs = append(c.fhs[:i], c.fhs[i+1:]...)
|
||||
return cfh.fh, nil
|
||||
|
||||
}
|
||||
}
|
||||
// fs.Debugf(nil, "CACHE MISS")
|
||||
// Just take the first one if not found
|
||||
cfh := c.fhs[0]
|
||||
c.fhs = c.fhs[1:]
|
||||
return cfh.fh, nil
|
||||
}
|
||||
|
||||
fh, err = c.node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open squashfs archive: %w", err)
|
||||
}
|
||||
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Close a vfs.Handle or return it to the pool
|
||||
//
|
||||
// off should be the offset the file handle would read from without seeking
|
||||
func (c *cache) close(fh vfs.Handle, off int64) {
|
||||
c.fhsMu.Lock()
|
||||
defer c.fhsMu.Unlock()
|
||||
|
||||
c.fhs = append(c.fhs, cacheHandle{
|
||||
offset: off,
|
||||
fh: fh,
|
||||
})
|
||||
}
|
||||
|
||||
// ReadAt reads len(p) bytes into p starting at offset off in the underlying
|
||||
// input source. It returns the number of bytes read (0 <= n <= len(p)) and any
|
||||
// error encountered.
|
||||
//
|
||||
// When ReadAt returns n < len(p), it returns a non-nil error explaining why
|
||||
// more bytes were not returned. In this respect, ReadAt is stricter than Read.
|
||||
//
|
||||
// Even if ReadAt returns n < len(p), it may use all of p as scratch
|
||||
// space during the call. If some data is available but not len(p) bytes,
|
||||
// ReadAt blocks until either all the data is available or an error occurs.
|
||||
// In this respect ReadAt is different from Read.
|
||||
//
|
||||
// If the n = len(p) bytes returned by ReadAt are at the end of the input
|
||||
// source, ReadAt may return either err == EOF or err == nil.
|
||||
//
|
||||
// If ReadAt is reading from an input source with a seek offset, ReadAt should
|
||||
// not affect nor be affected by the underlying seek offset.
|
||||
//
|
||||
// Clients of ReadAt can execute parallel ReadAt calls on the same input
|
||||
// source.
|
||||
//
|
||||
// Implementations must not retain p.
|
||||
func (c *cache) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
fh, err := c.open(off)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
defer func() {
|
||||
c.close(fh, off+int64(len(p)))
|
||||
}()
|
||||
// fs.Debugf(nil, "ReadAt(p[%d], off=%d, fh=%p)", len(p), off, fh)
|
||||
return fh.ReadAt(p, off)
|
||||
}
|
||||
|
||||
var errCacheNotImplemented = errors.New("internal error: squashfs cache doesn't implement method")
|
||||
|
||||
// WriteAt method dummy stub to satisfy interface
|
||||
func (c *cache) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
return 0, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Seek method dummy stub to satisfy interface
|
||||
func (c *cache) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, errCacheNotImplemented
|
||||
}
|
||||
|
|
@ -0,0 +1,446 @@
|
|||
// Package squashfs implements a squashfs archiver for the archive backend
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/go-diskfs/filesystem/squashfs"
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
func init() {
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: New,
|
||||
Extension: ".sqfs",
|
||||
})
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
sqfs *squashfs.FileSystem // interface to the squashfs
|
||||
c *cache
|
||||
node vfs.Node // squashfs file object - set if reading
|
||||
remote string // remote of the squashfs file object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.DefaultOpt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
c := newCache(node)
|
||||
|
||||
// FIXME blocksize
|
||||
sqfs, err := squashfs.Read(c, node.Size(), 0, 1024*1024)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read squashfs: %w", err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
sqfs: sqfs,
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
if prefix == "" {
|
||||
f.prefixSlash = ""
|
||||
}
|
||||
|
||||
singleObject := false
|
||||
|
||||
// Find the directory the root points to
|
||||
if f.root != "" && !strings.HasSuffix(root, "/") {
|
||||
native, err := f.toNative("")
|
||||
if err == nil {
|
||||
native = strings.TrimRight(native, "/")
|
||||
_, err := f.newObjectNative(native)
|
||||
if err == nil {
|
||||
// If it pointed to a file, find the directory above
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gsquashfs
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if singleObject {
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("Squashfs %q", f.name)
|
||||
}
|
||||
|
||||
// This turns a remote into a native path in the squashfs starting with a /
|
||||
func (f *Fs) toNative(remote string) (string, error) {
|
||||
native := strings.Trim(remote, "/")
|
||||
if f.prefix == "" {
|
||||
native = "/" + native
|
||||
} else if native == f.prefix {
|
||||
native = "/"
|
||||
} else if !strings.HasPrefix(native, f.prefixSlash) {
|
||||
return "", fmt.Errorf("internal error: %q doesn't start with prefix %q", native, f.prefixSlash)
|
||||
} else {
|
||||
native = native[len(f.prefix):]
|
||||
}
|
||||
if f.root != "" {
|
||||
native = "/" + f.root + native
|
||||
}
|
||||
return native, nil
|
||||
}
|
||||
|
||||
// Turn a (nativeDir, leaf) into a remote
|
||||
func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
|
||||
dir := nativeDir
|
||||
if f.root != "" {
|
||||
dir = strings.TrimPrefix(dir, "/"+f.root)
|
||||
}
|
||||
remote := f.prefixSlash + strings.Trim(path.Join(dir, leaf), "/")
|
||||
// fs.Debugf(nil, "dir = %q, remote=%q", dir, remote)
|
||||
return remote
|
||||
}
|
||||
|
||||
// Convert a FileInfo into an Object from native dir
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()),
|
||||
size: item.Size(),
|
||||
modTime: item.ModTime(),
|
||||
item: item,
|
||||
}
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
|
||||
nativeDir, err := f.toNative(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := f.sqfs.ReadDir(nativeDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
|
||||
}
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
for _, fi := range items {
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
var entry fs.DirEntry
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
|
||||
}
|
||||
if item.IsDir() {
|
||||
var remote = f.fromNative(nativeDir, item.Name())
|
||||
entry = fs.NewDir(remote, item.ModTime())
|
||||
} else {
|
||||
if item.Mode().IsRegular() {
|
||||
entry = f.objectFromFileInfo(nativeDir, item)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// newObjectNative finds the object at the native path passed in
|
||||
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
// get the path and filename
|
||||
dir, leaf := path.Split(nativePath)
|
||||
dir = strings.TrimRight(dir, "/")
|
||||
leaf = strings.Trim(leaf, "/")
|
||||
|
||||
// FIXME need to detect directory not found
|
||||
fis, err := f.sqfs.ReadDir(dir)
|
||||
if err != nil {
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if fi.Name() == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, item)
|
||||
break
|
||||
}
|
||||
}
|
||||
if o == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
|
||||
nativePath, err := f.toNative(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.newObjectNative(nativePath)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw squashfs file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
item squashfs.FileStat
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Turn a squashfs path into a full path for the parent Fs
|
||||
// func (o *Object) path(remote string) string {
|
||||
// return path.Join(o.fs.prefix, remote)
|
||||
// }
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.size
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Opening %q", remote)
|
||||
//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
|
||||
fh, err := o.item.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// discard data from start as necessary
|
||||
if offset > 0 {
|
||||
_, err = fh.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
|
||||
return readers.NewLimitedReadCloser(fh, limit), nil
|
||||
}
|
||||
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
|
|
@ -0,0 +1,385 @@
|
|||
// Package zip implements a zip archiver for the archive backend
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/dirtree"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
func init() {
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: New,
|
||||
Extension: ".zip",
|
||||
})
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
node vfs.Node // zip file object - set if reading
|
||||
remote string // remote of the zip file object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.DefaultOpt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
|
||||
// Read the contents of the zip file
|
||||
singleObject, err := f.readZip()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open zip file: %w", err)
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gzip
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if singleObject {
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("Zip %q", f.name)
|
||||
}
|
||||
|
||||
// readZip the zip file into f
|
||||
//
|
||||
// Returns singleObject=true if f.root points to a file
|
||||
func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
if f.node == nil {
|
||||
return singleObject, fs.ErrorDirNotFound
|
||||
}
|
||||
size := f.node.Size()
|
||||
if size < 0 {
|
||||
return singleObject, errors.New("can't read from zip file with unknown size")
|
||||
}
|
||||
r, err := f.node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return singleObject, fmt.Errorf("failed to open zip file: %w", err)
|
||||
}
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
|
||||
}
|
||||
dt := dirtree.New()
|
||||
for _, file := range zr.File {
|
||||
remote := strings.Trim(path.Clean(file.Name), "/")
|
||||
if remote == "." {
|
||||
remote = ""
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
continue
|
||||
}
|
||||
if remote == f.root {
|
||||
remote = ""
|
||||
} else {
|
||||
remote = strings.TrimPrefix(remote, f.root+"/")
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(file.Name, "/") {
|
||||
dir := fs.NewDir(remote, file.Modified)
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
}
|
||||
o := &Object{
|
||||
f: f,
|
||||
remote: remote,
|
||||
fh: &file.FileHeader,
|
||||
file: file,
|
||||
}
|
||||
dt.Add(o)
|
||||
if singleObject {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
//fs.Debugf(nil, "dt = %v", dt)
|
||||
return singleObject, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
// _, err = f.strip(dir)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
entries, ok := f.dt[dir]
|
||||
if !ok {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
if f.dt == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
_, entry := f.dt.Find(remote)
|
||||
if entry == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
o, ok := entry.(*Object)
|
||||
if !ok {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.CRC32)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw zip file
|
||||
type Object struct {
|
||||
f *Fs
|
||||
remote string
|
||||
fh *zip.FileHeader
|
||||
file *zip.File
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.f
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return int64(o.fh.UncompressedSize64)
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return o.fh.Modified
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
if ht == hash.CRC32 {
|
||||
// FIXME return empty CRC if writing
|
||||
if o.f.dt == nil {
|
||||
return "", nil
|
||||
}
|
||||
return fmt.Sprintf("%08x", o.fh.CRC32), nil
|
||||
}
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rc, err = o.file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// discard data from start as necessary
|
||||
if offset > 0 {
|
||||
_, err = io.CopyN(io.Discard, rc, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
return readers.NewLimitedReadCloser(rc, limit), nil
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
|
|
@ -2,3 +2,8 @@
|
|||
// their implementation these can't be imported by the VFS so need to
|
||||
// be mentioned in here, not backend/all.
|
||||
package all
|
||||
|
||||
import (
|
||||
// Active file systems
|
||||
_ "github.com/rclone/rclone/backend/archive"
|
||||
)
|
||||
|
|
|
|||
9
go.mod
9
go.mod
|
|
@ -2,6 +2,8 @@ module github.com/rclone/rclone
|
|||
|
||||
go 1.19
|
||||
|
||||
// replace github.com/diskfs/go-diskfs => /home/ncw/go/src/github.com/diskfs/go-diskfs
|
||||
|
||||
require (
|
||||
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0
|
||||
|
|
@ -38,7 +40,7 @@ require (
|
|||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/josephspurrier/goversioninfo v1.4.0
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
|
||||
github.com/klauspost/compress v1.17.2
|
||||
github.com/klauspost/compress v1.17.4
|
||||
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
|
||||
github.com/mattn/go-colorable v0.1.13
|
||||
|
|
@ -47,6 +49,7 @@ require (
|
|||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/moby/sys/mountinfo v0.6.2
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
|
||||
github.com/ncw/go-diskfs v1.4.1-0.20231223121205-c8a9a133379e
|
||||
github.com/ncw/swift/v2 v2.0.2
|
||||
github.com/oracle/oci-go-sdk/v65 v65.51.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
|
|
@ -57,7 +60,7 @@ require (
|
|||
github.com/rfjakob/eme v1.1.2
|
||||
github.com/rivo/uniseg v0.4.4
|
||||
github.com/shirou/gopsutil/v3 v3.23.9
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
|
||||
github.com/spf13/cobra v1.7.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
|
|
@ -147,6 +150,7 @@ require (
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
|
||||
|
|
@ -164,6 +168,7 @@ require (
|
|||
github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/willscott/go-nfs-client v0.0.0-20200605172546-271fa9065b33 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -353,8 +353,8 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC
|
|||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
|
||||
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
|
||||
|
|
@ -413,6 +413,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
|
|||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehdGZsm5LP7AhLOvjgbS8zN1tk=
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
|
||||
github.com/ncw/go-diskfs v1.4.1-0.20231223121205-c8a9a133379e h1:ZhVsEG0SrdZtnvQrjAA/GCkqyjd96HA/b0l+FZ7Smtw=
|
||||
github.com/ncw/go-diskfs v1.4.1-0.20231223121205-c8a9a133379e/go.mod h1:ONBG0/ef6gxAFEa9G550rV6+BzAI7uIN9lYG1BUcm50=
|
||||
github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
|
||||
github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
|
|
@ -428,6 +430,8 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ
|
|||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
|
|
@ -485,8 +489,8 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
|
|||
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
|
|
@ -529,6 +533,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f
|
|||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
|
||||
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
|
||||
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
|
|
|
|||
Loading…
Reference in New Issue