github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-24 15:43:45 +01:00
Code Releases Activity

b2: use new prefix and delimiter parameters in directory listings

Browse Source 
This makes --max-depth 1 directory listings much more efficient (it no
longer lists all the files) and simplifies the code, bringing it into
line with s3/swift/gcs

Fixes #944
This commit is contained in:
Nick Craig-Wood 2016-12-14 17:37:26 +00:00
parent 13b705e227
commit 215fd2a11d
3 changed files with 30 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
b2/api/types.go
View File

@ -154,6 +154,8 @@ type ListFileNamesRequest struct {
StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name.
MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000.
StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off.
Prefix string `json:"prefix,omitempty"` // optional - Files returned will be limited to those with the given prefix. Defaults to the empty string, which matches all files.
Delimiter string `json:"delimiter,omitempty"` // Files returned will be limited to those within the top folder, or any one subfolder. Defaults to NULL. Folder names will also be returned. The delimiter character will be used to "break" file names into folders.
}
// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions

81
b2/b2.go
View File

@ -465,34 +465,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
// sendDir works out given a lastDir and a remote which directories should be sent
func sendDir(lastDir string, remote string, level int) (dirNames []string, newLastDir string) {
dir := path.Dir(remote)
if dir == "." {
// No slashes - nothing to do!
return nil, lastDir
}
if dir == lastDir {
// Still in same directory
return nil, lastDir
}
newLastDir = lastDir
for {
slashes := strings.Count(dir, "/")
if !strings.HasPrefix(lastDir, dir) && slashes < level {
dirNames = append([]string{dir}, dirNames...)
}
if newLastDir == lastDir {
newLastDir = dir
}
dir = path.Dir(dir)
if dir == "." {
break
}
}
return dirNames, newLastDir
}
// listFn is called from list to handle an object
type listFn func(remote string, object *api.File, isDirectory bool) error
@ -503,6 +475,8 @@ var errEndList = errors.New("end list")
// list lists the objects into the function supplied from
// the bucket and root supplied
//
// dir is the starting directory, "" for root
//
// level is the depth to search to
//
// If prefix is set then startFileName is used as a prefix which all
@ -517,6 +491,14 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
if dir != "" {
root += dir + "/"
}
delimiter := ""
switch level {
case 1:
delimiter = "/"
case fs.MaxLevel:
default:
return fs.ErrorLevelNotSupported
}
bucketID, err := f.getBucketID()
if err != nil {
return err
@ -528,6 +510,8 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
var request = api.ListFileNamesRequest{
BucketID: bucketID,
MaxFileCount: chunkSize,
Prefix: root,
Delimiter: delimiter,
}
prefix = root + prefix
if prefix != "" {
@ -541,7 +525,6 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
if hidden {
opts.Path = "/b2_list_file_versions"
}
lastDir := dir
for {
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response)
@ -553,34 +536,26 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
for i := range response.Files {
file := &response.Files[i]
// Finish if file name no longer has prefix
if !strings.HasPrefix(file.Name, prefix) {
if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
return nil
}
remote := file.Name[len(f.root):]
slashes := strings.Count(remote, "/")
// Check if this file makes a new directories
var dirNames []string
dirNames, lastDir = sendDir(lastDir, remote, level)
for _, dirName := range dirNames {
err = fn(dirName, nil, true)
if err != nil {
if err == errEndList {
return nil
}
return err
}
if !strings.HasPrefix(file.Name, f.root) {
fs.Log(f, "Odd name received %q", file.Name)
continue
}
// Send the file
if slashes < level {
err = fn(remote, file, false)
if err != nil {
if err == errEndList {
return nil
}
return err
remote := file.Name[len(f.root):]
// Check for directory
isDirectory := level != 0 && strings.HasSuffix(remote, "/")
if isDirectory {
remote = remote[:len(remote)-1]
}
// Send object
err = fn(remote, file, isDirectory)
if err != nil {
if err == errEndList {
return nil
}
return err
}
}
// end if no NextFileName

114
b2/b2_internal_test.go
View File

@ -5,7 +5,6 @@ import (
"time"
"github.com/ncw/rclone/fstest"
"github.com/stretchr/testify/assert"
)
// Test b2 string encoding
@ -169,116 +168,3 @@ func TestParseTimeString(t *testing.T) {
}
}
func TestSendDir(t *testing.T) {
for _, test := range []struct {
lastDir string
remote string
level int
dirNames []string
newLastDir string
}{
{
lastDir: "",
remote: "test.txt",
level: 100,
dirNames: nil,
newLastDir: "",
},
{
lastDir: "",
remote: "potato/test.txt",
level: 100,
dirNames: []string{"potato"},
newLastDir: "potato",
},
{
lastDir: "potato",
remote: "potato/test.txt",
level: 100,
dirNames: nil,
newLastDir: "potato",
},
{
lastDir: "",
remote: "potato/sausage/test.txt",
level: 100,
dirNames: []string{"potato", "potato/sausage"},
newLastDir: "potato/sausage",
},
{
lastDir: "potato",
remote: "potato/sausage/test.txt",
level: 100,
dirNames: []string{"potato/sausage"},
newLastDir: "potato/sausage",
},
{
lastDir: "potato/sausage",
remote: "potato/sausage/test.txt",
level: 100,
dirNames: nil,
newLastDir: "potato/sausage",
},
{
lastDir: "",
remote: "a/b/c/d/e/f.txt",
level: 100,
dirNames: []string{"a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/e"},
newLastDir: "a/b/c/d/e",
},
{
lastDir: "a/b/c/d/e",
remote: "a/b/c/d/E/f.txt",
level: 100,
dirNames: []string{"a/b/c/d/E"},
newLastDir: "a/b/c/d/E",
},
{
lastDir: "a/b/c/d/e",
remote: "a/b/C/D/E/f.txt",
level: 100,
dirNames: []string{"a/b/C", "a/b/C/D", "a/b/C/D/E"},
newLastDir: "a/b/C/D/E",
},
{
lastDir: "a/b/c",
remote: "a/b/c/d/e/f.txt",
level: 100,
dirNames: []string{"a/b/c/d", "a/b/c/d/e"},
newLastDir: "a/b/c/d/e",
},
{
lastDir: "",
remote: "a/b/c/d/e/f.txt",
level: 1,
dirNames: []string{"a"},
newLastDir: "a/b/c/d/e",
},
{
lastDir: "a/b/c",
remote: "a/b/c/d/e/f.txt",
level: 1,
dirNames: nil,
newLastDir: "a/b/c/d/e",
},
{
lastDir: "",
remote: "a/b/c/d/e/f.txt",
level: 3,
dirNames: []string{"a", "a/b", "a/b/c"},
newLastDir: "a/b/c/d/e",
},
{
lastDir: "a/b/C/D/E",
remote: "a/b/c/d/e/f.txt",
level: 3,
dirNames: []string{"a/b/c"},
newLastDir: "a/b/c/d/e",
},
} {
dirNames, newLastDir := sendDir(test.lastDir, test.remote, test.level)
assert.Equal(t, test.dirNames, dirNames, "dirNames fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
assert.Equal(t, test.newLastDir, newLastDir, "newLastDir fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
}
}