mirror of
https://github.com/rclone/rclone
synced 2024-12-23 14:23:44 +01:00
b2: use new prefix and delimiter parameters in directory listings
This makes --max-depth 1 directory listings much more efficient (it no longer lists all the files) and simplifies the code, bringing it into line with s3/swift/gcs Fixes #944
This commit is contained in:
parent
13b705e227
commit
215fd2a11d
@ -154,6 +154,8 @@ type ListFileNamesRequest struct {
|
||||
StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name.
|
||||
MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000.
|
||||
StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off.
|
||||
Prefix string `json:"prefix,omitempty"` // optional - Files returned will be limited to those with the given prefix. Defaults to the empty string, which matches all files.
|
||||
Delimiter string `json:"delimiter,omitempty"` // Files returned will be limited to those within the top folder, or any one subfolder. Defaults to NULL. Folder names will also be returned. The delimiter character will be used to "break" file names into folders.
|
||||
}
|
||||
|
||||
// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions
|
||||
|
81
b2/b2.go
81
b2/b2.go
@ -465,34 +465,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
return f.newObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// sendDir works out given a lastDir and a remote which directories should be sent
|
||||
func sendDir(lastDir string, remote string, level int) (dirNames []string, newLastDir string) {
|
||||
dir := path.Dir(remote)
|
||||
if dir == "." {
|
||||
// No slashes - nothing to do!
|
||||
return nil, lastDir
|
||||
}
|
||||
if dir == lastDir {
|
||||
// Still in same directory
|
||||
return nil, lastDir
|
||||
}
|
||||
newLastDir = lastDir
|
||||
for {
|
||||
slashes := strings.Count(dir, "/")
|
||||
if !strings.HasPrefix(lastDir, dir) && slashes < level {
|
||||
dirNames = append([]string{dir}, dirNames...)
|
||||
}
|
||||
if newLastDir == lastDir {
|
||||
newLastDir = dir
|
||||
}
|
||||
dir = path.Dir(dir)
|
||||
if dir == "." {
|
||||
break
|
||||
}
|
||||
}
|
||||
return dirNames, newLastDir
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *api.File, isDirectory bool) error
|
||||
|
||||
@ -503,6 +475,8 @@ var errEndList = errors.New("end list")
|
||||
// list lists the objects into the function supplied from
|
||||
// the bucket and root supplied
|
||||
//
|
||||
// dir is the starting directory, "" for root
|
||||
//
|
||||
// level is the depth to search to
|
||||
//
|
||||
// If prefix is set then startFileName is used as a prefix which all
|
||||
@ -517,6 +491,14 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
|
||||
if dir != "" {
|
||||
root += dir + "/"
|
||||
}
|
||||
delimiter := ""
|
||||
switch level {
|
||||
case 1:
|
||||
delimiter = "/"
|
||||
case fs.MaxLevel:
|
||||
default:
|
||||
return fs.ErrorLevelNotSupported
|
||||
}
|
||||
bucketID, err := f.getBucketID()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -528,6 +510,8 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
|
||||
var request = api.ListFileNamesRequest{
|
||||
BucketID: bucketID,
|
||||
MaxFileCount: chunkSize,
|
||||
Prefix: root,
|
||||
Delimiter: delimiter,
|
||||
}
|
||||
prefix = root + prefix
|
||||
if prefix != "" {
|
||||
@ -541,7 +525,6 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
|
||||
if hidden {
|
||||
opts.Path = "/b2_list_file_versions"
|
||||
}
|
||||
lastDir := dir
|
||||
for {
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(&opts, &request, &response)
|
||||
@ -553,34 +536,26 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool,
|
||||
for i := range response.Files {
|
||||
file := &response.Files[i]
|
||||
// Finish if file name no longer has prefix
|
||||
if !strings.HasPrefix(file.Name, prefix) {
|
||||
if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
|
||||
return nil
|
||||
}
|
||||
remote := file.Name[len(f.root):]
|
||||
slashes := strings.Count(remote, "/")
|
||||
|
||||
// Check if this file makes a new directories
|
||||
var dirNames []string
|
||||
dirNames, lastDir = sendDir(lastDir, remote, level)
|
||||
for _, dirName := range dirNames {
|
||||
err = fn(dirName, nil, true)
|
||||
if err != nil {
|
||||
if err == errEndList {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !strings.HasPrefix(file.Name, f.root) {
|
||||
fs.Log(f, "Odd name received %q", file.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
// Send the file
|
||||
if slashes < level {
|
||||
err = fn(remote, file, false)
|
||||
if err != nil {
|
||||
if err == errEndList {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
remote := file.Name[len(f.root):]
|
||||
// Check for directory
|
||||
isDirectory := level != 0 && strings.HasSuffix(remote, "/")
|
||||
if isDirectory {
|
||||
remote = remote[:len(remote)-1]
|
||||
}
|
||||
// Send object
|
||||
err = fn(remote, file, isDirectory)
|
||||
if err != nil {
|
||||
if err == errEndList {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
// end if no NextFileName
|
||||
|
@ -5,7 +5,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Test b2 string encoding
|
||||
@ -169,116 +168,3 @@ func TestParseTimeString(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSendDir(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
lastDir string
|
||||
remote string
|
||||
level int
|
||||
dirNames []string
|
||||
newLastDir string
|
||||
}{
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "test.txt",
|
||||
level: 100,
|
||||
dirNames: nil,
|
||||
newLastDir: "",
|
||||
},
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "potato/test.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"potato"},
|
||||
newLastDir: "potato",
|
||||
},
|
||||
{
|
||||
lastDir: "potato",
|
||||
remote: "potato/test.txt",
|
||||
level: 100,
|
||||
dirNames: nil,
|
||||
newLastDir: "potato",
|
||||
},
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "potato/sausage/test.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"potato", "potato/sausage"},
|
||||
newLastDir: "potato/sausage",
|
||||
},
|
||||
{
|
||||
lastDir: "potato",
|
||||
remote: "potato/sausage/test.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"potato/sausage"},
|
||||
newLastDir: "potato/sausage",
|
||||
},
|
||||
{
|
||||
lastDir: "potato/sausage",
|
||||
remote: "potato/sausage/test.txt",
|
||||
level: 100,
|
||||
dirNames: nil,
|
||||
newLastDir: "potato/sausage",
|
||||
},
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"a", "a/b", "a/b/c", "a/b/c/d", "a/b/c/d/e"},
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
{
|
||||
lastDir: "a/b/c/d/e",
|
||||
remote: "a/b/c/d/E/f.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"a/b/c/d/E"},
|
||||
newLastDir: "a/b/c/d/E",
|
||||
},
|
||||
{
|
||||
lastDir: "a/b/c/d/e",
|
||||
remote: "a/b/C/D/E/f.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"a/b/C", "a/b/C/D", "a/b/C/D/E"},
|
||||
newLastDir: "a/b/C/D/E",
|
||||
},
|
||||
{
|
||||
lastDir: "a/b/c",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 100,
|
||||
dirNames: []string{"a/b/c/d", "a/b/c/d/e"},
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 1,
|
||||
dirNames: []string{"a"},
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
{
|
||||
lastDir: "a/b/c",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 1,
|
||||
dirNames: nil,
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
{
|
||||
lastDir: "",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 3,
|
||||
dirNames: []string{"a", "a/b", "a/b/c"},
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
{
|
||||
lastDir: "a/b/C/D/E",
|
||||
remote: "a/b/c/d/e/f.txt",
|
||||
level: 3,
|
||||
dirNames: []string{"a/b/c"},
|
||||
newLastDir: "a/b/c/d/e",
|
||||
},
|
||||
} {
|
||||
dirNames, newLastDir := sendDir(test.lastDir, test.remote, test.level)
|
||||
assert.Equal(t, test.dirNames, dirNames, "dirNames fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
|
||||
assert.Equal(t, test.newLastDir, newLastDir, "newLastDir fail for sendDir(%q,%q,%v)", test.lastDir, test.remote, test.level)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user