mirror of
https://github.com/rclone/rclone
synced 2024-12-23 14:23:44 +01:00
vfs: re-use the File objects when re-reading the directory
Make it so that d.items is never nil to simplify the code This should help with inconsistent reads when the source object changes.
This commit is contained in:
parent
7fb53a031c
commit
5e334eedd2
52
vfs/dir.go
52
vfs/dir.go
@ -24,7 +24,7 @@ type Dir struct {
|
|||||||
entry fs.Directory
|
entry fs.Directory
|
||||||
mu sync.Mutex // protects the following
|
mu sync.Mutex // protects the following
|
||||||
read time.Time // time directory entry last read
|
read time.Time // time directory entry last read
|
||||||
items map[string]Node // NB can be nil when directory not read yet
|
items map[string]Node // directory entries - can be empty but not nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
|
func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
|
||||||
@ -36,6 +36,7 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
|
|||||||
path: fsDir.Remote(),
|
path: fsDir.Remote(),
|
||||||
modTime: fsDir.ModTime(),
|
modTime: fsDir.ModTime(),
|
||||||
inode: newInode(),
|
inode: newInode(),
|
||||||
|
items: make(map[string]Node),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -111,7 +112,7 @@ func (d *Dir) ForgetPath(relativePath string) {
|
|||||||
d.walk(absPath, func(dir *Dir) {
|
d.walk(absPath, func(dir *Dir) {
|
||||||
fs.Debugf(dir.path, "forgetting directory cache")
|
fs.Debugf(dir.path, "forgetting directory cache")
|
||||||
dir.read = time.Time{}
|
dir.read = time.Time{}
|
||||||
dir.items = nil
|
dir.items = make(map[string]Node)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -122,13 +123,11 @@ func (d *Dir) ForgetPath(relativePath string) {
|
|||||||
func (d *Dir) walk(absPath string, fun func(*Dir)) {
|
func (d *Dir) walk(absPath string, fun func(*Dir)) {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
if d.items != nil {
|
|
||||||
for _, node := range d.items {
|
for _, node := range d.items {
|
||||||
if dir, ok := node.(*Dir); ok {
|
if dir, ok := node.(*Dir); ok {
|
||||||
dir.walk(absPath, fun)
|
dir.walk(absPath, fun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if d.path == absPath || absPath == "" || strings.HasPrefix(d.path, absPath+"/") {
|
if d.path == absPath || absPath == "" || strings.HasPrefix(d.path, absPath+"/") {
|
||||||
fun(d)
|
fun(d)
|
||||||
@ -153,25 +152,21 @@ func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) {
|
|||||||
// note that we add new objects rather than updating old ones
|
// note that we add new objects rather than updating old ones
|
||||||
func (d *Dir) addObject(node Node) {
|
func (d *Dir) addObject(node Node) {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
if d.items != nil {
|
|
||||||
d.items[node.Name()] = node
|
d.items[node.Name()] = node
|
||||||
}
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// delObject removes an object from the directory
|
// delObject removes an object from the directory
|
||||||
func (d *Dir) delObject(leaf string) {
|
func (d *Dir) delObject(leaf string) {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
if d.items != nil {
|
|
||||||
delete(d.items, leaf)
|
delete(d.items, leaf)
|
||||||
}
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// read the directory and sets d.items - must be called with the lock held
|
// read the directory and sets d.items - must be called with the lock held
|
||||||
func (d *Dir) _readDir() error {
|
func (d *Dir) _readDir() error {
|
||||||
when := time.Now()
|
when := time.Now()
|
||||||
if d.read.IsZero() || d.items == nil {
|
if d.read.IsZero() {
|
||||||
// fs.Debugf(d.path, "Reading directory")
|
// fs.Debugf(d.path, "Reading directory")
|
||||||
} else {
|
} else {
|
||||||
age := when.Sub(d.read)
|
age := when.Sub(d.read)
|
||||||
@ -187,39 +182,40 @@ func (d *Dir) _readDir() error {
|
|||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// NB when we re-read a directory after its cache has expired
|
|
||||||
// we drop the old files which should lead to correct
|
|
||||||
// behaviour but may not be very efficient.
|
|
||||||
|
|
||||||
// Keep a note of the previous contents of the directory
|
|
||||||
oldItems := d.items
|
|
||||||
|
|
||||||
// Cache the items by name
|
// Cache the items by name
|
||||||
d.items = make(map[string]Node, len(entries))
|
found := make(map[string]struct{})
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
|
name := path.Base(entry.Remote())
|
||||||
|
node := d.items[name]
|
||||||
|
found[name] = struct{}{}
|
||||||
switch item := entry.(type) {
|
switch item := entry.(type) {
|
||||||
case fs.Object:
|
case fs.Object:
|
||||||
obj := item
|
obj := item
|
||||||
name := path.Base(obj.Remote())
|
// Reuse old file value if it exists
|
||||||
d.items[name] = newFile(d, obj, name)
|
if file, ok := node.(*File); node != nil && ok {
|
||||||
|
file.setObjectNoUpdate(obj)
|
||||||
|
} else {
|
||||||
|
node = newFile(d, obj, name)
|
||||||
|
}
|
||||||
case fs.Directory:
|
case fs.Directory:
|
||||||
dir := item
|
dir := item
|
||||||
name := path.Base(dir.Remote())
|
// Reuse old dir value if it exists
|
||||||
// Use old dir value if it exists
|
if node == nil || !node.IsDir() {
|
||||||
if oldItems != nil {
|
node = newDir(d.vfs, d.f, d, dir)
|
||||||
if oldNode, ok := oldItems[name]; ok {
|
|
||||||
if oldNode.IsDir() {
|
|
||||||
d.items[name] = oldNode
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
d.items[name] = newDir(d.vfs, d.f, d, dir)
|
|
||||||
default:
|
default:
|
||||||
err = errors.Errorf("unknown type %T", item)
|
err = errors.Errorf("unknown type %T", item)
|
||||||
fs.Errorf(d, "readDir error: %v", err)
|
fs.Errorf(d, "readDir error: %v", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
d.items[name] = node
|
||||||
|
}
|
||||||
|
// delete unused entries
|
||||||
|
for name := range d.items {
|
||||||
|
if _, ok := found[name]; !ok {
|
||||||
|
delete(d.items, name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
d.read = when
|
d.read = when
|
||||||
return nil
|
return nil
|
||||||
|
@ -252,6 +252,14 @@ func (f *File) setObject(o fs.Object) {
|
|||||||
f.d.addObject(f)
|
f.d.addObject(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update the object but don't update the directory cache - for use by
|
||||||
|
// the directory cache
|
||||||
|
func (f *File) setObjectNoUpdate(o fs.Object) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.o = o
|
||||||
|
}
|
||||||
|
|
||||||
// Get the current fs.Object - may be nil
|
// Get the current fs.Object - may be nil
|
||||||
func (f *File) getObject() fs.Object {
|
func (f *File) getObject() fs.Object {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
|
Loading…
Reference in New Issue
Block a user