github
/
rclone
mirror of https://github.com/rclone/rclone
1
Fork 0
Code Releases Activity

box: make listings of heavily used directories more reliable #5545 

Before this change we uses limit/offset paging for directories in the
main directory listing routine and in the trash cleanup listing.

This switches to the new scheme of limit/marker which is more reliable
on a directory which is continuously changing. It has the disadvantage
that it doesn't tell us the total number of items available, however
that wasn't information rclone uses.
This commit is contained in:
Nick Craig-Wood 2021-08-20 10:13:33 +01:00
parent fc5d6c16b6
commit 308323e9c4
2 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/box/api/types.go
View File

@ -103,10 +103,11 @@ func (i *Item) ModTime() (t time.Time) {
// FolderItems is returned from the GetFolderItems call
type FolderItems struct {
TotalCount int `json:"total_count"`
Entries []Item `json:"entries"`
Offset int `json:"offset"`
Limit int `json:"limit"`
TotalCount int `json:"total_count"`
Entries []Item `json:"entries"`
Offset int `json:"offset"`
Limit int `json:"limit"`
NextMarker *string `json:"next_marker,omitempty"`
Order []struct {
By string `json:"by"`
Direction string `json:"direction"`

22
backend/box/box.go
View File

@ -578,10 +578,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
Parameters: fieldsValue(),
}
opts.Parameters.Set("limit", strconv.Itoa(listChunks))
offset := 0
opts.Parameters.Set("usemarker", "true")
var marker *string
OUTER:
for {
opts.Parameters.Set("offset", strconv.Itoa(offset))
if marker != nil {
opts.Parameters.Set("marker", *marker)
}
var result api.FolderItems
var resp *http.Response
@ -615,8 +618,8 @@ OUTER:
break OUTER
}
}
offset += result.Limit
if offset >= result.TotalCount {
marker = result.NextMarker
if marker == nil {
break
}
}
@ -1100,9 +1103,12 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
},
}
opts.Parameters.Set("limit", strconv.Itoa(listChunks))
offset := 0
opts.Parameters.Set("usemarker", "true")
var marker *string
for {
opts.Parameters.Set("offset", strconv.Itoa(offset))
if marker != nil {
opts.Parameters.Set("marker", *marker)
}
var result api.FolderItems
var resp *http.Response
@ -1125,8 +1131,8 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
continue
}
}
offset += result.Limit
if offset >= result.TotalCount {
marker = result.NextMarker
if marker == nil {
break
}
}