mirror of
https://github.com/rclone/rclone
synced 2024-12-23 14:23:44 +01:00
vfs: implement --vfs-cache-max-size to limit the total size of the cache
This commit is contained in:
Nick Craig-Wood
parent
fffdbb31f5
commit
a43ed567ee
125
vfs/cache.go
125
vfs/cache.go
@ -67,8 +67,9 @@ type cache struct {
|
||||
f fs.Fs // fs for the cache directory
|
||||
opt *Options // vfs Options
|
||||
root string // root of the cache directory
|
||||
itemMu sync.Mutex // protects the next two maps
|
||||
itemMu sync.Mutex // protects the following variables
|
||||
item map[string]*cacheItem // files/directories in the cache
|
||||
used int64 // total size of files in the cache
|
||||
}
|
||||
|
||||
// cacheItem is stored in the item map
|
||||
@ -76,6 +77,7 @@ type cacheItem struct {
|
||||
opens int // number of times file is open
|
||||
atime time.Time // last time file was accessed
|
||||
isFile bool // if this is a file or a directory
|
||||
size int64 // size of the cached item
|
||||
}
|
||||
|
||||
// newCacheItem returns an item for the cache
|
||||
@ -196,11 +198,13 @@ func (c *cache) get(name string) *cacheItem {
|
||||
return item
|
||||
}
|
||||
|
||||
// updateTime sets the atime of the name to that passed in if it is
|
||||
// updateStat sets the atime of the name to that passed in if it is
|
||||
// newer than the existing or there isn't an existing time.
|
||||
//
|
||||
// it also sets the size
|
||||
//
|
||||
// name should be a remote path not an osPath
|
||||
func (c *cache) updateTime(name string, when time.Time) {
|
||||
func (c *cache) updateStat(name string, when time.Time, size int64) {
|
||||
name = clean(name)
|
||||
c.itemMu.Lock()
|
||||
item, found := c._get(true, name)
|
||||
@ -208,6 +212,7 @@ func (c *cache) updateTime(name string, when time.Time) {
|
||||
fs.Debugf(name, "updateTime: setting atime to %v", when)
|
||||
item.atime = when
|
||||
}
|
||||
item.size = size
|
||||
c.itemMu.Unlock()
|
||||
}
|
||||
|
||||
@ -266,6 +271,12 @@ func (c *cache) _close(isFile bool, name string) {
|
||||
if item.opens < 0 {
|
||||
fs.Errorf(name, "cache: double close")
|
||||
}
|
||||
osPath := c.toOSPath(name)
|
||||
fi, err := os.Stat(osPath)
|
||||
// Update the size on close
|
||||
if err == nil && !fi.IsDir() {
|
||||
item.size = fi.Size()
|
||||
}
|
||||
if name == "" {
|
||||
break
|
||||
}
|
||||
@ -291,7 +302,7 @@ func (c *cache) remove(name string) {
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
fs.Errorf(name, "Failed to remove from cache: %v", err)
|
||||
} else {
|
||||
fs.Debugf(name, "Removed from cache")
|
||||
fs.Infof(name, "Removed from cache")
|
||||
}
|
||||
}
|
||||
|
||||
@ -338,26 +349,34 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error)
|
||||
})
|
||||
}
|
||||
|
||||
// updateAtimes walks the cache updating any atimes it finds
|
||||
func (c *cache) updateAtimes() error {
|
||||
return c.walk(func(osPath string, fi os.FileInfo, name string) error {
|
||||
// updateStats walks the cache updating any atimes and sizes it finds
|
||||
//
|
||||
// it also updates used
|
||||
func (c *cache) updateStats() error {
|
||||
var newUsed int64
|
||||
err := c.walk(func(osPath string, fi os.FileInfo, name string) error {
|
||||
if !fi.IsDir() {
|
||||
// Update the atime with that of the file
|
||||
atime := times.Get(fi).AccessTime()
|
||||
c.updateTime(name, atime)
|
||||
c.updateStat(name, atime, fi.Size())
|
||||
newUsed += fi.Size()
|
||||
} else {
|
||||
c.cacheDir(name)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
c.itemMu.Lock()
|
||||
c.used = newUsed
|
||||
c.itemMu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// purgeOld gets rid of any files that are over age
|
||||
func (c *cache) purgeOld(maxAge time.Duration) {
|
||||
c._purgeOld(maxAge, c.remove, c.removeDir)
|
||||
c._purgeOld(maxAge, c.remove)
|
||||
}
|
||||
|
||||
func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), removeDir func(name string) bool) {
|
||||
func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
|
||||
c.itemMu.Lock()
|
||||
defer c.itemMu.Unlock()
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
@ -373,7 +392,16 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove
|
||||
}
|
||||
}
|
||||
}
|
||||
// now find any empty directories
|
||||
}
|
||||
|
||||
// Purge any empty directories
|
||||
func (c *cache) purgeEmptyDirs() {
|
||||
c._purgeEmptyDirs(c.removeDir)
|
||||
}
|
||||
|
||||
func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) {
|
||||
c.itemMu.Lock()
|
||||
defer c.itemMu.Unlock()
|
||||
var dirs []string
|
||||
for name, item := range c.item {
|
||||
if !item.isFile && item.opens == 0 {
|
||||
@ -391,6 +419,56 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string), remove
|
||||
}
|
||||
}
|
||||
|
||||
// This is a cacheItem with a name for sorting
|
||||
type cacheNamedItem struct {
|
||||
name string
|
||||
item *cacheItem
|
||||
}
|
||||
type cacheNamedItems []cacheNamedItem
|
||||
|
||||
func (v cacheNamedItems) Len() int { return len(v) }
|
||||
func (v cacheNamedItems) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
|
||||
func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j].item.atime) }
|
||||
|
||||
// Remove any files that are over quota starting from the
|
||||
// oldest first
|
||||
func (c *cache) purgeOverQuota(quota int64) {
|
||||
c._purgeOverQuota(quota, c.remove)
|
||||
}
|
||||
|
||||
func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
|
||||
c.itemMu.Lock()
|
||||
defer c.itemMu.Unlock()
|
||||
|
||||
if quota <= 0 || c.used < quota {
|
||||
return
|
||||
}
|
||||
|
||||
var items cacheNamedItems
|
||||
|
||||
// Make a slice of unused files
|
||||
for name, item := range c.item {
|
||||
if item.isFile && item.opens == 0 {
|
||||
items = append(items, cacheNamedItem{
|
||||
name: name,
|
||||
item: item,
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Sort(items)
|
||||
|
||||
// Remove items until the quota is OK
|
||||
for _, item := range items {
|
||||
if c.used < quota {
|
||||
break
|
||||
}
|
||||
remove(item.name)
|
||||
// Remove the entry
|
||||
delete(c.item, item.name)
|
||||
c.used -= item.item.size
|
||||
}
|
||||
}
|
||||
|
||||
// clean empties the cache of stuff if it can
|
||||
func (c *cache) clean() {
|
||||
// Cache may be empty so end
|
||||
@ -399,17 +477,32 @@ func (c *cache) clean() {
|
||||
return
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Cleaning the cache")
|
||||
c.itemMu.Lock()
|
||||
oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
|
||||
c.itemMu.Unlock()
|
||||
|
||||
// first walk the FS to update the atimes
|
||||
err = c.updateAtimes()
|
||||
// first walk the FS to update the atimes and sizes
|
||||
err = c.updateStats()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err)
|
||||
}
|
||||
|
||||
// Now remove any files that are over age and any empty
|
||||
// directories
|
||||
// Remove any files that are over age
|
||||
c.purgeOld(c.opt.CacheMaxAge)
|
||||
|
||||
// Now remove any files that are over quota starting from the
|
||||
// oldest first
|
||||
c.purgeOverQuota(int64(c.opt.CacheMaxSize))
|
||||
|
||||
// Remove any empty directories
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
// Stats
|
||||
c.itemMu.Lock()
|
||||
newItems, newUsed := len(c.item), fs.SizeSuffix(c.used)
|
||||
c.itemMu.Unlock()
|
||||
|
||||
fs.Infof(nil, "Cleaned the cache: objects %d (was %d), total size %v (was %v)", newItems, oldItems, newUsed, oldUsed)
|
||||
}
|
||||
|
||||
// cleaner calls clean at regular intervals
|
||||
|
||||
@ -51,7 +51,7 @@ func itemAsString(c *cache) []string {
|
||||
defer c.itemMu.Unlock()
|
||||
var out []string
|
||||
for name, item := range c.item {
|
||||
out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens))
|
||||
out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d size=%d", name, item.isFile, item.opens, item.size))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
@ -79,7 +79,7 @@ func TestCacheNew(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "potato", filepath.Base(p))
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
fi, err := os.Stat(filepath.Dir(p))
|
||||
@ -95,26 +95,26 @@ func TestCacheNew(t *testing.T) {
|
||||
// updateTime
|
||||
//.. before
|
||||
t1 := time.Now().Add(-60 * time.Minute)
|
||||
c.updateTime("potato", t1)
|
||||
c.updateStat("potato", t1, 0)
|
||||
item = c.get("potato")
|
||||
assert.NotEqual(t, t1, item.atime)
|
||||
assert.Equal(t, 0, item.opens)
|
||||
//..after
|
||||
t2 := time.Now().Add(60 * time.Minute)
|
||||
c.updateTime("potato", t2)
|
||||
c.updateStat("potato", t2, 0)
|
||||
item = c.get("potato")
|
||||
assert.Equal(t, t2, item.atime)
|
||||
assert.Equal(t, 0, item.opens)
|
||||
|
||||
// open
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
c.open("/potato")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
item = c.get("potato")
|
||||
assert.WithinDuration(t, time.Now(), item.atime, time.Second)
|
||||
@ -132,11 +132,11 @@ func TestCacheNew(t *testing.T) {
|
||||
// updateAtimes
|
||||
item = c.get("potato")
|
||||
item.atime = time.Now().Add(-24 * time.Hour)
|
||||
err = c.updateAtimes()
|
||||
err = c.updateStats()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=5`,
|
||||
}, itemAsString(c))
|
||||
item = c.get("potato")
|
||||
assert.Equal(t, atime, item.atime)
|
||||
@ -146,11 +146,11 @@ func TestCacheNew(t *testing.T) {
|
||||
c.itemMu.Lock()
|
||||
delete(c.item, "potato") // remove from cache
|
||||
c.itemMu.Unlock()
|
||||
err = c.updateAtimes()
|
||||
err = c.updateStats()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=0 size=5`,
|
||||
}, itemAsString(c))
|
||||
item = c.get("potato")
|
||||
assert.Equal(t, atime, item.atime)
|
||||
@ -165,18 +165,18 @@ func TestCacheNew(t *testing.T) {
|
||||
|
||||
// close
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=5`,
|
||||
}, itemAsString(c))
|
||||
c.updateTime("potato", t2)
|
||||
c.updateStat("potato", t2, 6)
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=6`,
|
||||
}, itemAsString(c))
|
||||
c.close("potato/")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="potato" isFile=true opens=0 size=5`,
|
||||
}, itemAsString(c))
|
||||
item = c.get("potato")
|
||||
assert.WithinDuration(t, time.Now(), item.atime, time.Second)
|
||||
@ -216,23 +216,23 @@ func TestCacheOpens(t *testing.T) {
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
c.open("potato")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
c.open("potato")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=2`,
|
||||
`name="potato" isFile=true opens=2`,
|
||||
`name="" isFile=false opens=2 size=0`,
|
||||
`name="potato" isFile=true opens=2 size=0`,
|
||||
}, itemAsString(c))
|
||||
c.close("potato")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
c.close("potato")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
c.open("potato")
|
||||
@ -240,34 +240,34 @@ func TestCacheOpens(t *testing.T) {
|
||||
c.open("a/b/c/d/e/two")
|
||||
c.open("a/b/c/d/e/f/three")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=4`,
|
||||
`name="a" isFile=false opens=3`,
|
||||
`name="a/b" isFile=false opens=3`,
|
||||
`name="a/b/c" isFile=false opens=3`,
|
||||
`name="a/b/c/d" isFile=false opens=3`,
|
||||
`name="a/b/c/d/e" isFile=false opens=2`,
|
||||
`name="a/b/c/d/e/f" isFile=false opens=1`,
|
||||
`name="a/b/c/d/e/f/three" isFile=true opens=1`,
|
||||
`name="a/b/c/d/e/two" isFile=true opens=1`,
|
||||
`name="a/b/c/d/one" isFile=true opens=1`,
|
||||
`name="potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=4 size=0`,
|
||||
`name="a" isFile=false opens=3 size=0`,
|
||||
`name="a/b" isFile=false opens=3 size=0`,
|
||||
`name="a/b/c" isFile=false opens=3 size=0`,
|
||||
`name="a/b/c/d" isFile=false opens=3 size=0`,
|
||||
`name="a/b/c/d/e" isFile=false opens=2 size=0`,
|
||||
`name="a/b/c/d/e/f" isFile=false opens=1 size=0`,
|
||||
`name="a/b/c/d/e/f/three" isFile=true opens=1 size=0`,
|
||||
`name="a/b/c/d/e/two" isFile=true opens=1 size=0`,
|
||||
`name="a/b/c/d/one" isFile=true opens=1 size=0`,
|
||||
`name="potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
c.close("potato")
|
||||
c.close("a/b/c/d/one")
|
||||
c.close("a/b/c/d/e/two")
|
||||
c.close("a/b/c//d/e/f/three")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="a" isFile=false opens=0`,
|
||||
`name="a/b" isFile=false opens=0`,
|
||||
`name="a/b/c" isFile=false opens=0`,
|
||||
`name="a/b/c/d" isFile=false opens=0`,
|
||||
`name="a/b/c/d/e" isFile=false opens=0`,
|
||||
`name="a/b/c/d/e/f" isFile=false opens=0`,
|
||||
`name="a/b/c/d/e/f/three" isFile=true opens=0`,
|
||||
`name="a/b/c/d/e/two" isFile=true opens=0`,
|
||||
`name="a/b/c/d/one" isFile=true opens=0`,
|
||||
`name="potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="a" isFile=false opens=0 size=0`,
|
||||
`name="a/b" isFile=false opens=0 size=0`,
|
||||
`name="a/b/c" isFile=false opens=0 size=0`,
|
||||
`name="a/b/c/d" isFile=false opens=0 size=0`,
|
||||
`name="a/b/c/d/e" isFile=false opens=0 size=0`,
|
||||
`name="a/b/c/d/e/f" isFile=false opens=0 size=0`,
|
||||
`name="a/b/c/d/e/f/three" isFile=true opens=0 size=0`,
|
||||
`name="a/b/c/d/e/two" isFile=true opens=0 size=0`,
|
||||
`name="a/b/c/d/one" isFile=true opens=0 size=0`,
|
||||
`name="potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
}
|
||||
|
||||
@ -289,9 +289,9 @@ func TestCacheOpenMkdir(t *testing.T) {
|
||||
c.open("sub/potato")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="sub" isFile=false opens=1`,
|
||||
`name="sub/potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="sub" isFile=false opens=1 size=0`,
|
||||
`name="sub/potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// mkdir
|
||||
@ -299,9 +299,9 @@ func TestCacheOpenMkdir(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "potato", filepath.Base(p))
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=1`,
|
||||
`name="sub" isFile=false opens=1`,
|
||||
`name="sub/potato" isFile=true opens=1`,
|
||||
`name="" isFile=false opens=1 size=0`,
|
||||
`name="sub" isFile=false opens=1 size=0`,
|
||||
`name="sub/potato" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// test directory exists
|
||||
@ -321,13 +321,14 @@ func TestCacheOpenMkdir(t *testing.T) {
|
||||
c.close("sub/potato")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="sub" isFile=false opens=0`,
|
||||
`name="sub/potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// clean the cache
|
||||
c.purgeOld(-10 * time.Second)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
@ -350,24 +351,24 @@ func TestCacheCacheDir(t *testing.T) {
|
||||
|
||||
c.cacheDir("dir")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="dir" isFile=false opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="dir" isFile=false opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
c.cacheDir("dir/sub")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="dir" isFile=false opens=0`,
|
||||
`name="dir/sub" isFile=false opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="dir" isFile=false opens=0 size=0`,
|
||||
`name="dir/sub" isFile=false opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
c.cacheDir("dir/sub2/subsub2")
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="dir" isFile=false opens=0`,
|
||||
`name="dir/sub" isFile=false opens=0`,
|
||||
`name="dir/sub2" isFile=false opens=0`,
|
||||
`name="dir/sub2/subsub2" isFile=false opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="dir" isFile=false opens=0 size=0`,
|
||||
`name="dir/sub" isFile=false opens=0 size=0`,
|
||||
`name="dir/sub2" isFile=false opens=0 size=0`,
|
||||
`name="dir/sub2/subsub2" isFile=false opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
}
|
||||
|
||||
@ -395,7 +396,8 @@ func TestCachePurgeOld(t *testing.T) {
|
||||
}
|
||||
|
||||
removed = nil
|
||||
c._purgeOld(-10*time.Second, removeFile, removeDir)
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
c.open("sub/dir2/potato2")
|
||||
@ -404,17 +406,18 @@ func TestCachePurgeOld(t *testing.T) {
|
||||
c.open("sub/dir/potato")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=2`,
|
||||
`name="sub" isFile=false opens=2`,
|
||||
`name="sub/dir" isFile=false opens=2`,
|
||||
`name="sub/dir/potato" isFile=true opens=2`,
|
||||
`name="sub/dir2" isFile=false opens=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=2 size=0`,
|
||||
`name="sub" isFile=false opens=2 size=0`,
|
||||
`name="sub/dir" isFile=false opens=2 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=2 size=0`,
|
||||
`name="sub/dir2" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile, removeDir)
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir2/potato2",
|
||||
"sub/dir2/",
|
||||
@ -424,33 +427,36 @@ func TestCachePurgeOld(t *testing.T) {
|
||||
|
||||
removed = nil
|
||||
removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile, removeDir)
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
c.close("sub/dir/potato")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="sub" isFile=false opens=0`,
|
||||
`name="sub/dir" isFile=false opens=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
removedDir = false
|
||||
c._purgeOld(10*time.Second, removeFile, removeDir)
|
||||
c._purgeOld(10*time.Second, removeFile)
|
||||
c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0`,
|
||||
`name="sub" isFile=false opens=0`,
|
||||
`name="sub/dir" isFile=false opens=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0`,
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile, removeDir)
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
"sub/dir/",
|
||||
@ -460,3 +466,157 @@ func TestCachePurgeOld(t *testing.T) {
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
}
|
||||
|
||||
func TestCachePurgeOverQuota(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Disable the cache cleaner as it interferes with these tests
|
||||
opt := DefaultOpt
|
||||
opt.CachePollInterval = 0
|
||||
c, err := newCache(ctx, r.Fremote, &opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Test funcs
|
||||
var removed []string
|
||||
remove := func(name string) {
|
||||
removed = append(removed, name)
|
||||
c.remove(name)
|
||||
}
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(-1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(0, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
// Make some test files
|
||||
c.open("sub/dir/potato")
|
||||
p, err := c.mkdir("sub/dir/potato")
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(p, []byte("hello"), 0600)
|
||||
require.NoError(t, err)
|
||||
|
||||
p, err = c.mkdir("sub/dir2/potato2")
|
||||
c.open("sub/dir2/potato2")
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(p, []byte("hello2"), 0600)
|
||||
require.NoError(t, err)
|
||||
// make it definitely after
|
||||
t1 := time.Now().Add(10 * time.Second)
|
||||
c.updateStat("sub/dir2/potato2", t1, 0)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=2 size=0`,
|
||||
`name="sub" isFile=false opens=2 size=0`,
|
||||
`name="sub/dir" isFile=false opens=1 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=1 size=0`,
|
||||
`name="sub/dir2" isFile=false opens=1 size=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Check nothing removed
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
// Close the files
|
||||
c.close("sub/dir/potato")
|
||||
c.close("sub/dir2/potato2")
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0 size=5`,
|
||||
`name="sub/dir2" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Update the stats to read the total size
|
||||
err = c.updateStats()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(11), c.used)
|
||||
|
||||
// Check only potato removed to get below quota
|
||||
removed = nil
|
||||
c._purgeOverQuota(10, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(6), c.used)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir2" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Put potato back
|
||||
c.open("sub/dir/potato")
|
||||
p, err = c.mkdir("sub/dir/potato")
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(p, []byte("hello"), 0600)
|
||||
require.NoError(t, err)
|
||||
c.close("sub/dir/potato")
|
||||
// make it definitely after
|
||||
t2 := t1.Add(20 * time.Second)
|
||||
c.updateStat("sub/dir/potato", t2, 5)
|
||||
|
||||
// Update the stats to read the total size
|
||||
err = c.updateStats()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(11), c.used)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0 size=5`,
|
||||
`name="sub/dir2" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Check only potato2 removed to get below quota
|
||||
removed = nil
|
||||
c._purgeOverQuota(10, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir2/potato2",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(5), c.used)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="" isFile=false opens=0 size=0`,
|
||||
`name="sub" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir" isFile=false opens=0 size=0`,
|
||||
`name="sub/dir/potato" isFile=true opens=0 size=5`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Now purge everything
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(0), c.used)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
// Check nothing left behind
|
||||
c.clean()
|
||||
assert.Equal(t, int64(0), c.used)
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
}
|
||||
|
||||
@ -60,6 +60,7 @@ may find that you need one or the other or both.
|
||||
--vfs-cache-max-age duration Max age of objects in the cache. (default 1h0m0s)
|
||||
--vfs-cache-mode string Cache mode off|minimal|writes|full (default "off")
|
||||
--vfs-cache-poll-interval duration Interval to poll the cache for stale objects. (default 1m0s)
|
||||
--vfs-cache-max-size int Max total size of objects in the cache. (default off)
|
||||
|
||||
If run with ` + "`-vv`" + ` rclone will print the location of the file cache. The
|
||||
files are stored in the user cache file area which is OS dependent but
|
||||
@ -75,6 +76,11 @@ closed so if rclone is quit or dies with open files then these won't
|
||||
get written back to the remote. However they will still be in the on
|
||||
disk cache.
|
||||
|
||||
If using --vfs-cache-max-size note that the cache may exceed this size
|
||||
for two reasons. Firstly because it is only checked every
|
||||
--vfs-cache-poll-interval. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
#### --vfs-cache-mode off
|
||||
|
||||
In this mode the cache will read directly from the remote and write
|
||||
|
||||
@ -50,6 +50,7 @@ var DefaultOpt = Options{
|
||||
CachePollInterval: 60 * time.Second,
|
||||
ChunkSize: 128 * fs.MebiByte,
|
||||
ChunkSizeLimit: -1,
|
||||
CacheMaxSize: -1,
|
||||
}
|
||||
|
||||
// Node represents either a directory (*Dir) or a file (*File)
|
||||
@ -196,6 +197,7 @@ type Options struct {
|
||||
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
|
||||
CacheMode CacheMode
|
||||
CacheMaxAge time.Duration
|
||||
CacheMaxSize fs.SizeSuffix
|
||||
CachePollInterval time.Duration
|
||||
}
|
||||
|
||||
|
||||
@ -27,6 +27,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
|
||||
flags.FVarP(flagSet, &Opt.CacheMode, "vfs-cache-mode", "", "Cache mode off|minimal|writes|full")
|
||||
flags.DurationVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", Opt.CachePollInterval, "Interval to poll the cache for stale objects.")
|
||||
flags.DurationVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", Opt.CacheMaxAge, "Max age of objects in the cache.")
|
||||
flags.FVarP(flagSet, &Opt.CacheMaxSize, "vfs-cache-max-size", "", "Max total size of objects in the cache.")
|
||||
flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks.")
|
||||
flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited.")
|
||||
flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user