mirror of https://github.com/rclone/rclone
4016 lines
122 KiB
Go
4016 lines
122 KiB
Go
// Package drive interfaces with the Google Drive object storage system
|
|
package drive
|
|
|
|
// FIXME need to deal with some corner cases
|
|
// * multiple files with the same name
|
|
// * files can be in multiple directories
|
|
// * can have directory loops
|
|
// * files with / in name
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/tls"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"mime"
|
|
"net/http"
|
|
"path"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"text/template"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/cache"
|
|
"github.com/rclone/rclone/fs/config"
|
|
"github.com/rclone/rclone/fs/config/configmap"
|
|
"github.com/rclone/rclone/fs/config/configstruct"
|
|
"github.com/rclone/rclone/fs/config/obscure"
|
|
"github.com/rclone/rclone/fs/filter"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/rclone/rclone/fs/fshttp"
|
|
"github.com/rclone/rclone/fs/fspath"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fs/walk"
|
|
"github.com/rclone/rclone/lib/dircache"
|
|
"github.com/rclone/rclone/lib/encoder"
|
|
"github.com/rclone/rclone/lib/env"
|
|
"github.com/rclone/rclone/lib/oauthutil"
|
|
"github.com/rclone/rclone/lib/pacer"
|
|
"github.com/rclone/rclone/lib/readers"
|
|
"golang.org/x/oauth2"
|
|
"golang.org/x/oauth2/google"
|
|
drive_v2 "google.golang.org/api/drive/v2"
|
|
drive "google.golang.org/api/drive/v3"
|
|
"google.golang.org/api/googleapi"
|
|
"google.golang.org/api/option"
|
|
)
|
|
|
|
// Constants
|
|
const (
|
|
rcloneClientID = "202264815644.apps.googleusercontent.com"
|
|
rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
|
|
driveFolderType = "application/vnd.google-apps.folder"
|
|
shortcutMimeType = "application/vnd.google-apps.shortcut"
|
|
shortcutMimeTypeDangling = "application/vnd.google-apps.shortcut.dangling" // synthetic mime type for internal use
|
|
timeFormatIn = time.RFC3339
|
|
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
|
defaultMinSleep = fs.Duration(100 * time.Millisecond)
|
|
defaultBurst = 100
|
|
defaultExportExtensions = "docx,xlsx,pptx,svg"
|
|
scopePrefix = "https://www.googleapis.com/auth/"
|
|
defaultScope = "drive"
|
|
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
|
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
|
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
|
defaultChunkSize = 8 * fs.Mebi
|
|
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
|
listRGrouping = 50 // number of IDs to search at once when using ListR
|
|
listRInputBuffer = 1000 // size of input buffer when using ListR
|
|
defaultXDGIcon = "text-html"
|
|
)
|
|
|
|
// Globals
|
|
var (
|
|
// Description of how to auth for this app
|
|
driveConfig = &oauth2.Config{
|
|
Scopes: []string{scopePrefix + "drive"},
|
|
Endpoint: google.Endpoint,
|
|
ClientID: rcloneClientID,
|
|
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
|
RedirectURL: oauthutil.RedirectURL,
|
|
}
|
|
_mimeTypeToExtensionDuplicates = map[string]string{
|
|
"application/x-vnd.oasis.opendocument.presentation": ".odp",
|
|
"application/x-vnd.oasis.opendocument.spreadsheet": ".ods",
|
|
"application/x-vnd.oasis.opendocument.text": ".odt",
|
|
"image/jpg": ".jpg",
|
|
"image/x-bmp": ".bmp",
|
|
"image/x-png": ".png",
|
|
"text/rtf": ".rtf",
|
|
}
|
|
_mimeTypeToExtension = map[string]string{
|
|
"application/epub+zip": ".epub",
|
|
"application/json": ".json",
|
|
"application/msword": ".doc",
|
|
"application/pdf": ".pdf",
|
|
"application/rtf": ".rtf",
|
|
"application/vnd.ms-excel": ".xls",
|
|
"application/vnd.oasis.opendocument.presentation": ".odp",
|
|
"application/vnd.oasis.opendocument.spreadsheet": ".ods",
|
|
"application/vnd.oasis.opendocument.text": ".odt",
|
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
|
"application/x-msmetafile": ".wmf",
|
|
"application/zip": ".zip",
|
|
"image/bmp": ".bmp",
|
|
"image/jpeg": ".jpg",
|
|
"image/pjpeg": ".pjpeg",
|
|
"image/png": ".png",
|
|
"image/svg+xml": ".svg",
|
|
"text/csv": ".csv",
|
|
"text/html": ".html",
|
|
"text/plain": ".txt",
|
|
"text/tab-separated-values": ".tsv",
|
|
}
|
|
_mimeTypeToExtensionLinks = map[string]string{
|
|
"application/x-link-desktop": ".desktop",
|
|
"application/x-link-html": ".link.html",
|
|
"application/x-link-url": ".url",
|
|
"application/x-link-webloc": ".webloc",
|
|
}
|
|
_mimeTypeCustomTransform = map[string]string{
|
|
"application/vnd.google-apps.script+json": "application/json",
|
|
}
|
|
_mimeTypeToXDGLinkIcons = map[string]string{
|
|
"application/vnd.google-apps.document": "x-office-document",
|
|
"application/vnd.google-apps.drawing": "x-office-drawing",
|
|
"application/vnd.google-apps.presentation": "x-office-presentation",
|
|
"application/vnd.google-apps.spreadsheet": "x-office-spreadsheet",
|
|
}
|
|
fetchFormatsOnce sync.Once // make sure we fetch the export/import formats only once
|
|
_exportFormats map[string][]string // allowed export MIME type conversions
|
|
_importFormats map[string][]string // allowed import MIME type conversions
|
|
templatesOnce sync.Once // parse link templates only once
|
|
_linkTemplates map[string]*template.Template // available link types
|
|
)
|
|
|
|
// Parse the scopes option returning a slice of scopes
|
|
func driveScopes(scopesString string) (scopes []string) {
|
|
if scopesString == "" {
|
|
scopesString = defaultScope
|
|
}
|
|
for _, scope := range strings.Split(scopesString, ",") {
|
|
scope = strings.TrimSpace(scope)
|
|
scopes = append(scopes, scopePrefix+scope)
|
|
}
|
|
return scopes
|
|
}
|
|
|
|
// Returns true if one of the scopes was "drive.appfolder"
|
|
func driveScopesContainsAppFolder(scopes []string) bool {
|
|
for _, scope := range scopes {
|
|
if scope == scopePrefix+"drive.appfolder" {
|
|
return true
|
|
}
|
|
|
|
}
|
|
return false
|
|
}
|
|
|
|
func driveOAuthOptions() []fs.Option {
|
|
opts := []fs.Option{}
|
|
for _, opt := range oauthutil.SharedOptions {
|
|
if opt.Name == config.ConfigClientID {
|
|
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
|
|
}
|
|
opts = append(opts, opt)
|
|
}
|
|
return opts
|
|
}
|
|
|
|
// Register with Fs
|
|
func init() {
|
|
fs.Register(&fs.RegInfo{
|
|
Name: "drive",
|
|
Description: "Google Drive",
|
|
NewFs: NewFs,
|
|
CommandHelp: commandHelp,
|
|
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
|
// Parse config into Options struct
|
|
opt := new(Options)
|
|
err := configstruct.Set(m, opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
|
|
}
|
|
|
|
switch config.State {
|
|
case "":
|
|
// Fill in the scopes
|
|
driveConfig.Scopes = driveScopes(opt.Scope)
|
|
|
|
// Set the root_folder_id if using drive.appfolder
|
|
if driveScopesContainsAppFolder(driveConfig.Scopes) {
|
|
m.Set("root_folder_id", "appDataFolder")
|
|
}
|
|
|
|
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
|
|
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
|
|
OAuth2Config: driveConfig,
|
|
})
|
|
}
|
|
return fs.ConfigGoto("teamdrive")
|
|
case "teamdrive":
|
|
if opt.TeamDriveID == "" {
|
|
return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
|
|
}
|
|
return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
|
|
case "teamdrive_ok":
|
|
if config.Result == "false" {
|
|
m.Set("team_drive", "")
|
|
return nil, nil
|
|
}
|
|
return fs.ConfigGoto("teamdrive_config")
|
|
case "teamdrive_change":
|
|
if config.Result == "false" {
|
|
return nil, nil
|
|
}
|
|
return fs.ConfigGoto("teamdrive_config")
|
|
case "teamdrive_config":
|
|
f, err := newFs(ctx, name, "", m)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to make Fs to list Shared Drives: %w", err)
|
|
}
|
|
teamDrives, err := f.listTeamDrives(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(teamDrives) == 0 {
|
|
return fs.ConfigError("", "No Shared Drives found in your account")
|
|
}
|
|
return fs.ConfigChoose("teamdrive_final", "config_team_drive", "Shared Drive", len(teamDrives), func(i int) (string, string) {
|
|
teamDrive := teamDrives[i]
|
|
return teamDrive.Id, teamDrive.Name
|
|
})
|
|
case "teamdrive_final":
|
|
driveID := config.Result
|
|
m.Set("team_drive", driveID)
|
|
m.Set("root_folder_id", "")
|
|
opt.TeamDriveID = driveID
|
|
opt.RootFolderID = ""
|
|
return nil, nil
|
|
}
|
|
return nil, fmt.Errorf("unknown state %q", config.State)
|
|
},
|
|
Options: append(driveOAuthOptions(), []fs.Option{{
|
|
Name: "scope",
|
|
Help: "Scope that rclone should use when requesting access from drive.",
|
|
Examples: []fs.OptionExample{{
|
|
Value: "drive",
|
|
Help: "Full access all files, excluding Application Data Folder.",
|
|
}, {
|
|
Value: "drive.readonly",
|
|
Help: "Read-only access to file metadata and file contents.",
|
|
}, {
|
|
Value: "drive.file",
|
|
Help: "Access to files created by rclone only.\nThese are visible in the drive website.\nFile authorization is revoked when the user deauthorizes the app.",
|
|
}, {
|
|
Value: "drive.appfolder",
|
|
Help: "Allows read and write access to the Application Data folder.\nThis is not visible in the drive website.",
|
|
}, {
|
|
Value: "drive.metadata.readonly",
|
|
Help: "Allows read-only access to file metadata but\ndoes not allow any access to read or download file content.",
|
|
}},
|
|
}, {
|
|
Name: "root_folder_id",
|
|
Help: `ID of the root folder.
|
|
Leave blank normally.
|
|
|
|
Fill in to access "Computers" folders (see docs), or for rclone to use
|
|
a non root folder as its starting point.
|
|
`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "service_account_file",
|
|
Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
|
|
}, {
|
|
Name: "service_account_credentials",
|
|
Help: "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
|
Hide: fs.OptionHideConfigurator,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "team_drive",
|
|
Help: "ID of the Shared Drive (Team Drive).",
|
|
Hide: fs.OptionHideConfigurator,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "auth_owner_only",
|
|
Default: false,
|
|
Help: "Only consider files owned by the authenticated user.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "use_trash",
|
|
Default: true,
|
|
Help: "Send files to the trash instead of deleting permanently.\n\nDefaults to true, namely sending files to the trash.\nUse `--drive-use-trash=false` to delete files permanently instead.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "copy_shortcut_content",
|
|
Default: false,
|
|
Help: `Server side copy contents of shortcuts instead of the shortcut.
|
|
|
|
When doing server side copies, normally rclone will copy shortcuts as
|
|
shortcuts.
|
|
|
|
If this flag is used then rclone will copy the contents of shortcuts
|
|
rather than shortcuts themselves when doing server side copies.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "skip_gdocs",
|
|
Default: false,
|
|
Help: "Skip google documents in all listings.\n\nIf given, gdocs practically become invisible to rclone.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "skip_checksum_gphotos",
|
|
Default: false,
|
|
Help: `Skip MD5 checksum on Google photos and videos only.
|
|
|
|
Use this if you get checksum errors when transferring Google photos or
|
|
videos.
|
|
|
|
Setting this flag will cause Google photos and videos to return a
|
|
blank MD5 checksum.
|
|
|
|
Google photos are identified by being in the "photos" space.
|
|
|
|
Corrupted checksums are caused by Google modifying the image/video but
|
|
not updating the checksum.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "shared_with_me",
|
|
Default: false,
|
|
Help: `Only show files that are shared with me.
|
|
|
|
Instructs rclone to operate on your "Shared with me" folder (where
|
|
Google Drive lets you access the files and folders others have shared
|
|
with you).
|
|
|
|
This works both with the "list" (lsd, lsl, etc.) and the "copy"
|
|
commands (copy, sync, etc.), and with all other commands too.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "trashed_only",
|
|
Default: false,
|
|
Help: "Only show files that are in the trash.\n\nThis will show trashed files in their original directory structure.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "starred_only",
|
|
Default: false,
|
|
Help: "Only show files that are starred.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "formats",
|
|
Default: "",
|
|
Help: "Deprecated: See export_formats.",
|
|
Advanced: true,
|
|
Hide: fs.OptionHideConfigurator,
|
|
}, {
|
|
Name: "export_formats",
|
|
Default: defaultExportExtensions,
|
|
Help: "Comma separated list of preferred formats for downloading Google docs.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "import_formats",
|
|
Default: "",
|
|
Help: "Comma separated list of preferred formats for uploading Google docs.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "allow_import_name_change",
|
|
Default: false,
|
|
Help: "Allow the filetype to change when uploading Google docs.\n\nE.g. file.doc to file.docx. This will confuse sync and reupload every time.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "use_created_date",
|
|
Default: false,
|
|
Help: `Use file created date instead of modified date.
|
|
|
|
Useful when downloading data and you want the creation date used in
|
|
place of the last modified date.
|
|
|
|
**WARNING**: This flag may have some unexpected consequences.
|
|
|
|
When uploading to your drive all files will be overwritten unless they
|
|
haven't been modified since their creation. And the inverse will occur
|
|
while downloading. This side effect can be avoided by using the
|
|
"--checksum" flag.
|
|
|
|
This feature was implemented to retain photos capture date as recorded
|
|
by google photos. You will first need to check the "Create a Google
|
|
Photos folder" option in your google drive settings. You can then copy
|
|
or move the photos locally and use the date the image was taken
|
|
(created) set as the modification date.`,
|
|
Advanced: true,
|
|
Hide: fs.OptionHideConfigurator,
|
|
}, {
|
|
Name: "use_shared_date",
|
|
Default: false,
|
|
Help: `Use date file was shared instead of modified date.
|
|
|
|
Note that, as with "--drive-use-created-date", this flag may have
|
|
unexpected consequences when uploading/downloading files.
|
|
|
|
If both this flag and "--drive-use-created-date" are set, the created
|
|
date is used.`,
|
|
Advanced: true,
|
|
Hide: fs.OptionHideConfigurator,
|
|
}, {
|
|
Name: "list_chunk",
|
|
Default: 1000,
|
|
Help: "Size of listing chunk 100-1000, 0 to disable.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "impersonate",
|
|
Default: "",
|
|
Help: `Impersonate this user when using a service account.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "alternate_export",
|
|
Default: false,
|
|
Help: "Deprecated: No longer needed.",
|
|
Hide: fs.OptionHideBoth,
|
|
}, {
|
|
Name: "upload_cutoff",
|
|
Default: defaultChunkSize,
|
|
Help: "Cutoff for switching to chunked upload.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "chunk_size",
|
|
Default: defaultChunkSize,
|
|
Help: `Upload chunk size.
|
|
|
|
Must a power of 2 >= 256k.
|
|
|
|
Making this larger will improve performance, but note that each chunk
|
|
is buffered in memory one per transfer.
|
|
|
|
Reducing this will reduce memory usage but decrease performance.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "acknowledge_abuse",
|
|
Default: false,
|
|
Help: `Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
|
|
|
|
If downloading a file returns the error "This file has been identified
|
|
as malware or spam and cannot be downloaded" with the error code
|
|
"cannotDownloadAbusiveFile" then supply this flag to rclone to
|
|
indicate you acknowledge the risks of downloading the file and rclone
|
|
will download it anyway.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "keep_revision_forever",
|
|
Default: false,
|
|
Help: "Keep new head revision of each file forever.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "size_as_quota",
|
|
Default: false,
|
|
Help: `Show sizes as storage quota usage, not actual size.
|
|
|
|
Show the size of a file as the storage quota used. This is the
|
|
current version plus any older versions that have been set to keep
|
|
forever.
|
|
|
|
**WARNING**: This flag may have some unexpected consequences.
|
|
|
|
It is not recommended to set this flag in your config - the
|
|
recommended usage is using the flag form --drive-size-as-quota when
|
|
doing rclone ls/lsl/lsf/lsjson/etc only.
|
|
|
|
If you do use this flag for syncing (not recommended) then you will
|
|
need to use --ignore size also.`,
|
|
Advanced: true,
|
|
Hide: fs.OptionHideConfigurator,
|
|
}, {
|
|
Name: "v2_download_min_size",
|
|
Default: fs.SizeSuffix(-1),
|
|
Help: "If Object's are greater, use drive v2 API to download.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "pacer_min_sleep",
|
|
Default: defaultMinSleep,
|
|
Help: "Minimum time to sleep between API calls.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "pacer_burst",
|
|
Default: defaultBurst,
|
|
Help: "Number of API calls to allow without sleeping.",
|
|
Advanced: true,
|
|
}, {
|
|
Name: "server_side_across_configs",
|
|
Default: false,
|
|
Help: `Allow server-side operations (e.g. copy) to work across different drive configs.
|
|
|
|
This can be useful if you wish to do a server-side copy between two
|
|
different Google drives. Note that this isn't enabled by default
|
|
because it isn't easy to tell if it will work between any two
|
|
configurations.`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "disable_http2",
|
|
Default: true,
|
|
Help: `Disable drive using http2.
|
|
|
|
There is currently an unsolved issue with the google drive backend and
|
|
HTTP/2. HTTP/2 is therefore disabled by default for the drive backend
|
|
but can be re-enabled here. When the issue is solved this flag will
|
|
be removed.
|
|
|
|
See: https://github.com/rclone/rclone/issues/3631
|
|
|
|
`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "stop_on_upload_limit",
|
|
Default: false,
|
|
Help: `Make upload limit errors be fatal.
|
|
|
|
At the time of writing it is only possible to upload 750 GiB of data to
|
|
Google Drive a day (this is an undocumented limit). When this limit is
|
|
reached Google Drive produces a slightly different error message. When
|
|
this flag is set it causes these errors to be fatal. These will stop
|
|
the in-progress sync.
|
|
|
|
Note that this detection is relying on error message strings which
|
|
Google don't document so it may break in the future.
|
|
|
|
See: https://github.com/rclone/rclone/issues/3857
|
|
`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "stop_on_download_limit",
|
|
Default: false,
|
|
Help: `Make download limit errors be fatal.
|
|
|
|
At the time of writing it is only possible to download 10 TiB of data from
|
|
Google Drive a day (this is an undocumented limit). When this limit is
|
|
reached Google Drive produces a slightly different error message. When
|
|
this flag is set it causes these errors to be fatal. These will stop
|
|
the in-progress sync.
|
|
|
|
Note that this detection is relying on error message strings which
|
|
Google don't document so it may break in the future.
|
|
`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: "skip_shortcuts",
|
|
Help: `If set skip shortcut files.
|
|
|
|
Normally rclone dereferences shortcut files making them appear as if
|
|
they are the original file (see [the shortcuts section](#shortcuts)).
|
|
If this flag is set then rclone will ignore shortcut files completely.
|
|
`,
|
|
Advanced: true,
|
|
Default: false,
|
|
}, {
|
|
Name: "skip_dangling_shortcuts",
|
|
Help: `If set skip dangling shortcut files.
|
|
|
|
If this is set then rclone will not show any dangling shortcuts in listings.
|
|
`,
|
|
Advanced: true,
|
|
Default: false,
|
|
}, {
|
|
Name: "resource_key",
|
|
Help: `Resource key for accessing a link-shared file.
|
|
|
|
If you need to access files shared with a link like this
|
|
|
|
https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing
|
|
|
|
Then you will need to use the first part "XXX" as the "root_folder_id"
|
|
and the second part "YYY" as the "resource_key" otherwise you will get
|
|
404 not found errors when trying to access the directory.
|
|
|
|
See: https://developers.google.com/drive/api/guides/resource-keys
|
|
|
|
This resource key requirement only applies to a subset of old files.
|
|
|
|
Note also that opening the folder once in the web interface (with the
|
|
user you've authenticated rclone with) seems to be enough so that the
|
|
resource key is no needed.
|
|
`,
|
|
Advanced: true,
|
|
}, {
|
|
Name: config.ConfigEncoding,
|
|
Help: config.ConfigEncodingHelp,
|
|
Advanced: true,
|
|
// Encode invalid UTF-8 bytes as json doesn't handle them properly.
|
|
// Don't encode / as it's a valid name character in drive.
|
|
Default: encoder.EncodeInvalidUtf8,
|
|
}}...),
|
|
})
|
|
|
|
// register duplicate MIME types first
|
|
// this allows them to be used with mime.ExtensionsByType() but
|
|
// mime.TypeByExtension() will return the later registered MIME type
|
|
for _, m := range []map[string]string{
|
|
_mimeTypeToExtensionDuplicates, _mimeTypeToExtension, _mimeTypeToExtensionLinks,
|
|
} {
|
|
for mimeType, extension := range m {
|
|
if err := mime.AddExtensionType(extension, mimeType); err != nil {
|
|
fs.Errorf("Failed to register MIME type %q: %v", mimeType, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Options defines the configuration for this backend
|
|
type Options struct {
|
|
Scope string `config:"scope"`
|
|
RootFolderID string `config:"root_folder_id"`
|
|
ServiceAccountFile string `config:"service_account_file"`
|
|
ServiceAccountCredentials string `config:"service_account_credentials"`
|
|
TeamDriveID string `config:"team_drive"`
|
|
AuthOwnerOnly bool `config:"auth_owner_only"`
|
|
UseTrash bool `config:"use_trash"`
|
|
CopyShortcutContent bool `config:"copy_shortcut_content"`
|
|
SkipGdocs bool `config:"skip_gdocs"`
|
|
SkipChecksumGphotos bool `config:"skip_checksum_gphotos"`
|
|
SharedWithMe bool `config:"shared_with_me"`
|
|
TrashedOnly bool `config:"trashed_only"`
|
|
StarredOnly bool `config:"starred_only"`
|
|
Extensions string `config:"formats"`
|
|
ExportExtensions string `config:"export_formats"`
|
|
ImportExtensions string `config:"import_formats"`
|
|
AllowImportNameChange bool `config:"allow_import_name_change"`
|
|
UseCreatedDate bool `config:"use_created_date"`
|
|
UseSharedDate bool `config:"use_shared_date"`
|
|
ListChunk int64 `config:"list_chunk"`
|
|
Impersonate string `config:"impersonate"`
|
|
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
|
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
|
AcknowledgeAbuse bool `config:"acknowledge_abuse"`
|
|
KeepRevisionForever bool `config:"keep_revision_forever"`
|
|
SizeAsQuota bool `config:"size_as_quota"`
|
|
V2DownloadMinSize fs.SizeSuffix `config:"v2_download_min_size"`
|
|
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
|
|
PacerBurst int `config:"pacer_burst"`
|
|
ServerSideAcrossConfigs bool `config:"server_side_across_configs"`
|
|
DisableHTTP2 bool `config:"disable_http2"`
|
|
StopOnUploadLimit bool `config:"stop_on_upload_limit"`
|
|
StopOnDownloadLimit bool `config:"stop_on_download_limit"`
|
|
SkipShortcuts bool `config:"skip_shortcuts"`
|
|
SkipDanglingShortcuts bool `config:"skip_dangling_shortcuts"`
|
|
ResourceKey string `config:"resource_key"`
|
|
Enc encoder.MultiEncoder `config:"encoding"`
|
|
}
|
|
|
|
// Fs represents a remote drive server
|
|
type Fs struct {
|
|
name string // name of this remote
|
|
root string // the path we are working on
|
|
opt Options // parsed options
|
|
ci *fs.ConfigInfo // global config
|
|
features *fs.Features // optional features
|
|
svc *drive.Service // the connection to the drive server
|
|
v2Svc *drive_v2.Service // used to create download links for the v2 api
|
|
client *http.Client // authorized client
|
|
rootFolderID string // the id of the root folder
|
|
dirCache *dircache.DirCache // Map of directory path to directory id
|
|
lastQuery string // Last query string to check in unit tests
|
|
pacer *fs.Pacer // To pace the API calls
|
|
exportExtensions []string // preferred extensions to download docs
|
|
importMimeTypes []string // MIME types to convert to docs
|
|
isTeamDrive bool // true if this is a team drive
|
|
fileFields googleapi.Field // fields to fetch file info with
|
|
m configmap.Mapper
|
|
grouping int32 // number of IDs to search at once in ListR - read with atomic
|
|
listRmu *sync.Mutex // protects listRempties
|
|
listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
|
|
dirResourceKeys *sync.Map // map directory ID to resource key
|
|
}
|
|
|
|
type baseObject struct {
|
|
fs *Fs // what this object is part of
|
|
remote string // The remote path
|
|
id string // Drive Id of this object
|
|
modifiedDate string // RFC3339 time it was last modified
|
|
mimeType string // The object MIME type
|
|
bytes int64 // size of the object
|
|
parents []string // IDs of the parent directories
|
|
resourceKey *string // resourceKey is needed for link shared objects
|
|
}
|
|
type documentObject struct {
|
|
baseObject
|
|
url string // Download URL of this object
|
|
documentMimeType string // the original document MIME type
|
|
extLen int // The length of the added export extension
|
|
}
|
|
type linkObject struct {
|
|
baseObject
|
|
content []byte // The file content generated by a link template
|
|
extLen int // The length of the added export extension
|
|
}
|
|
|
|
// Object describes a drive object
|
|
type Object struct {
|
|
baseObject
|
|
url string // Download URL of this object
|
|
md5sum string // md5sum of the object
|
|
v2Download bool // generate v2 download link ondemand
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
func (f *Fs) Name() string {
|
|
return f.name
|
|
}
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
func (f *Fs) Root() string {
|
|
return f.root
|
|
}
|
|
|
|
// String converts this Fs to a string
|
|
func (f *Fs) String() string {
|
|
return fmt.Sprintf("Google drive root '%s'", f.root)
|
|
}
|
|
|
|
// Features returns the optional features of this Fs
|
|
func (f *Fs) Features() *fs.Features {
|
|
return f.features
|
|
}
|
|
|
|
// shouldRetry determines whether a given err rates being retried
|
|
func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
|
if fserrors.ContextError(ctx, &err) {
|
|
return false, err
|
|
}
|
|
if err == nil {
|
|
return false, nil
|
|
}
|
|
if fserrors.ShouldRetry(err) {
|
|
return true, err
|
|
}
|
|
switch gerr := err.(type) {
|
|
case *googleapi.Error:
|
|
if gerr.Code >= 500 && gerr.Code < 600 {
|
|
// All 5xx errors should be retried
|
|
return true, err
|
|
}
|
|
if len(gerr.Errors) > 0 {
|
|
reason := gerr.Errors[0].Reason
|
|
if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
|
|
if f.opt.StopOnUploadLimit && gerr.Errors[0].Message == "User rate limit exceeded." {
|
|
fs.Errorf(f, "Received upload limit error: %v", err)
|
|
return false, fserrors.FatalError(err)
|
|
}
|
|
return true, err
|
|
} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
|
|
fs.Errorf(f, "Received download limit error: %v", err)
|
|
return false, fserrors.FatalError(err)
|
|
} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
|
|
fs.Errorf(f, "Received upload limit error: %v", err)
|
|
return false, fserrors.FatalError(err)
|
|
} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
|
|
fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
|
|
return false, fserrors.FatalError(err)
|
|
}
|
|
}
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
// parseParse parses a drive 'url'
|
|
func parseDrivePath(path string) (root string, err error) {
|
|
root = strings.Trim(path, "/")
|
|
return
|
|
}
|
|
|
|
// User function to process a File item from list
|
|
//
|
|
// Should return true to finish processing
|
|
type listFn func(*drive.File) bool
|
|
|
|
func containsString(slice []string, s string) bool {
|
|
for _, e := range slice {
|
|
if e == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// getFile returns drive.File for the ID passed and fields passed in
|
|
func (f *Fs) getFile(ctx context.Context, ID string, fields googleapi.Field) (info *drive.File, err error) {
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
info, err = f.svc.Files.Get(ID).
|
|
Fields(fields).
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
return info, err
|
|
}
|
|
|
|
// getRootID returns the canonical ID for the "root" ID
|
|
func (f *Fs) getRootID(ctx context.Context) (string, error) {
|
|
info, err := f.getFile(ctx, "root", "id")
|
|
if err != nil {
|
|
return "", fmt.Errorf("couldn't find root directory ID: %w", err)
|
|
}
|
|
return info.Id, nil
|
|
}
|
|
|
|
// Lists the directory required calling the user function on each item found
|
|
//
|
|
// If the user fn ever returns true then it early exits with found = true
|
|
//
|
|
// Search params: https://developers.google.com/drive/search-parameters
|
|
func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directoriesOnly, filesOnly, trashedOnly, includeAll bool, fn listFn) (found bool, err error) {
|
|
var query []string
|
|
if !includeAll {
|
|
q := "trashed=" + strconv.FormatBool(trashedOnly)
|
|
if f.opt.TrashedOnly {
|
|
q = fmt.Sprintf("(mimeType='%s' or %s)", driveFolderType, q)
|
|
}
|
|
query = append(query, q)
|
|
}
|
|
|
|
// Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents)
|
|
// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
|
|
// If we need to list file inside those shared folders, we must search it without sharedWithMe
|
|
parentsQuery := bytes.NewBufferString("(")
|
|
var resourceKeys []string
|
|
for _, dirID := range dirIDs {
|
|
if dirID == "" {
|
|
continue
|
|
}
|
|
if parentsQuery.Len() > 1 {
|
|
_, _ = parentsQuery.WriteString(" or ")
|
|
}
|
|
if (f.opt.SharedWithMe || f.opt.StarredOnly) && dirID == f.rootFolderID {
|
|
if f.opt.SharedWithMe {
|
|
_, _ = parentsQuery.WriteString("sharedWithMe=true")
|
|
}
|
|
if f.opt.StarredOnly {
|
|
if f.opt.SharedWithMe {
|
|
_, _ = parentsQuery.WriteString(" and ")
|
|
}
|
|
_, _ = parentsQuery.WriteString("starred=true")
|
|
}
|
|
} else {
|
|
_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
|
|
}
|
|
resourceKey, hasResourceKey := f.dirResourceKeys.Load(dirID)
|
|
if hasResourceKey {
|
|
resourceKeys = append(resourceKeys, fmt.Sprintf("%s/%s", dirID, resourceKey))
|
|
}
|
|
}
|
|
resourceKeysHeader := strings.Join(resourceKeys, ",")
|
|
if parentsQuery.Len() > 1 {
|
|
_ = parentsQuery.WriteByte(')')
|
|
query = append(query, parentsQuery.String())
|
|
}
|
|
var stems []string
|
|
if title != "" {
|
|
searchTitle := f.opt.Enc.FromStandardName(title)
|
|
// Escaping the backslash isn't documented but seems to work
|
|
searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
|
|
searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
|
|
|
|
var titleQuery bytes.Buffer
|
|
_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
|
|
if !directoriesOnly && !f.opt.SkipGdocs {
|
|
// If the search title has an extension that is in the export extensions add a search
|
|
// for the filename without the extension.
|
|
// Assume that export extensions don't contain escape sequences.
|
|
for _, ext := range f.exportExtensions {
|
|
if strings.HasSuffix(searchTitle, ext) {
|
|
stems = append(stems, title[:len(title)-len(ext)])
|
|
_, _ = fmt.Fprintf(&titleQuery, " or name='%s'", searchTitle[:len(searchTitle)-len(ext)])
|
|
}
|
|
}
|
|
}
|
|
_ = titleQuery.WriteByte(')')
|
|
query = append(query, titleQuery.String())
|
|
}
|
|
if directoriesOnly {
|
|
query = append(query, fmt.Sprintf("(mimeType='%s' or mimeType='%s')", driveFolderType, shortcutMimeType))
|
|
}
|
|
if filesOnly {
|
|
query = append(query, fmt.Sprintf("mimeType!='%s'", driveFolderType))
|
|
}
|
|
|
|
// Constrain query using filter if this remote is a sync/copy/walk source.
|
|
if fi, use := filter.GetConfig(ctx), filter.GetUseFilter(ctx); fi != nil && use {
|
|
queryByTime := func(op string, tm time.Time) {
|
|
if tm.IsZero() {
|
|
return
|
|
}
|
|
// https://developers.google.com/drive/api/v3/ref-search-terms#operators
|
|
// Query times use RFC 3339 format, default timezone is UTC
|
|
timeStr := tm.UTC().Format("2006-01-02T15:04:05")
|
|
term := fmt.Sprintf("(modifiedTime %s '%s' or mimeType = '%s')", op, timeStr, driveFolderType)
|
|
query = append(query, term)
|
|
}
|
|
queryByTime(">=", fi.ModTimeFrom)
|
|
queryByTime("<=", fi.ModTimeTo)
|
|
}
|
|
|
|
list := f.svc.Files.List()
|
|
queryString := strings.Join(query, " and ")
|
|
if queryString != "" {
|
|
list.Q(queryString)
|
|
// fs.Debugf(f, "list query: %q", queryString)
|
|
}
|
|
f.lastQuery = queryString // for unit tests
|
|
|
|
if f.opt.ListChunk > 0 {
|
|
list.PageSize(f.opt.ListChunk)
|
|
}
|
|
list.SupportsAllDrives(true)
|
|
list.IncludeItemsFromAllDrives(true)
|
|
if f.isTeamDrive && !f.opt.SharedWithMe {
|
|
list.DriveId(f.opt.TeamDriveID)
|
|
list.Corpora("drive")
|
|
}
|
|
// If using appDataFolder then need to add Spaces
|
|
if f.rootFolderID == "appDataFolder" {
|
|
list.Spaces("appDataFolder")
|
|
}
|
|
// Add resource Keys if necessary
|
|
if resourceKeysHeader != "" {
|
|
list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader)
|
|
}
|
|
|
|
fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields)
|
|
|
|
OUTER:
|
|
for {
|
|
var files *drive.FileList
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return false, fmt.Errorf("couldn't list directory: %w", err)
|
|
}
|
|
if files.IncompleteSearch {
|
|
fs.Errorf(f, "search result INCOMPLETE")
|
|
}
|
|
for _, item := range files.Files {
|
|
item.Name = f.opt.Enc.ToStandardName(item.Name)
|
|
if isShortcut(item) {
|
|
// ignore shortcuts if directed
|
|
if f.opt.SkipShortcuts {
|
|
continue
|
|
}
|
|
// skip file shortcuts if directory only
|
|
if directoriesOnly && item.ShortcutDetails.TargetMimeType != driveFolderType {
|
|
continue
|
|
}
|
|
// skip directory shortcuts if file only
|
|
if filesOnly && item.ShortcutDetails.TargetMimeType == driveFolderType {
|
|
continue
|
|
}
|
|
item, err = f.resolveShortcut(ctx, item)
|
|
if err != nil {
|
|
return false, fmt.Errorf("list: %w", err)
|
|
}
|
|
// leave the dangling shortcut out of the listings
|
|
// we've already logged about the dangling shortcut in resolveShortcut
|
|
if f.opt.SkipDanglingShortcuts && item.MimeType == shortcutMimeTypeDangling {
|
|
continue
|
|
}
|
|
}
|
|
// Check the case of items is correct since
|
|
// the `=` operator is case insensitive.
|
|
if title != "" && title != item.Name {
|
|
found := false
|
|
for _, stem := range stems {
|
|
if stem == item.Name {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
_, exportName, _, _ := f.findExportFormat(ctx, item)
|
|
if exportName == "" || exportName != title {
|
|
continue
|
|
}
|
|
}
|
|
if fn(item) {
|
|
found = true
|
|
break OUTER
|
|
}
|
|
}
|
|
if files.NextPageToken == "" {
|
|
break
|
|
}
|
|
list.PageToken(files.NextPageToken)
|
|
}
|
|
return
|
|
}
|
|
|
|
// Returns true of x is a power of 2 or zero
|
|
func isPowerOfTwo(x int64) bool {
|
|
switch {
|
|
case x == 0:
|
|
return true
|
|
case x < 0:
|
|
return false
|
|
default:
|
|
return (x & (x - 1)) == 0
|
|
}
|
|
}
|
|
|
|
// add a charset parameter to all text/* MIME types
|
|
func fixMimeType(mimeTypeIn string) string {
|
|
if mimeTypeIn == "" {
|
|
return ""
|
|
}
|
|
mediaType, param, err := mime.ParseMediaType(mimeTypeIn)
|
|
if err != nil {
|
|
return mimeTypeIn
|
|
}
|
|
mimeTypeOut := mimeTypeIn
|
|
if strings.HasPrefix(mediaType, "text/") && param["charset"] == "" {
|
|
param["charset"] = "utf-8"
|
|
mimeTypeOut = mime.FormatMediaType(mediaType, param)
|
|
}
|
|
if mimeTypeOut == "" {
|
|
panic(fmt.Errorf("unable to fix MIME type %q", mimeTypeIn))
|
|
}
|
|
return mimeTypeOut
|
|
}
|
|
func fixMimeTypeMap(in map[string][]string) (out map[string][]string) {
|
|
out = make(map[string][]string, len(in))
|
|
for k, v := range in {
|
|
for i, mt := range v {
|
|
v[i] = fixMimeType(mt)
|
|
}
|
|
out[fixMimeType(k)] = v
|
|
}
|
|
return out
|
|
}
|
|
func isInternalMimeType(mimeType string) bool {
|
|
return strings.HasPrefix(mimeType, "application/vnd.google-apps.")
|
|
}
|
|
func isLinkMimeType(mimeType string) bool {
|
|
return strings.HasPrefix(mimeType, "application/x-link-")
|
|
}
|
|
|
|
// parseExtensions parses a list of comma separated extensions
|
|
// into a list of unique extensions with leading "." and a list of associated MIME types
|
|
func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, err error) {
|
|
for _, extensionText := range extensionsIn {
|
|
for _, extension := range strings.Split(extensionText, ",") {
|
|
extension = strings.ToLower(strings.TrimSpace(extension))
|
|
if extension == "" {
|
|
continue
|
|
}
|
|
if len(extension) > 0 && extension[0] != '.' {
|
|
extension = "." + extension
|
|
}
|
|
mt := mime.TypeByExtension(extension)
|
|
if mt == "" {
|
|
return extensions, mimeTypes, fmt.Errorf("couldn't find MIME type for extension %q", extension)
|
|
}
|
|
if !containsString(extensions, extension) {
|
|
extensions = append(extensions, extension)
|
|
mimeTypes = append(mimeTypes, mt)
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// getClient makes an http client according to the options
|
|
func getClient(ctx context.Context, opt *Options) *http.Client {
|
|
t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
|
|
if opt.DisableHTTP2 {
|
|
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
|
|
}
|
|
})
|
|
return &http.Client{
|
|
Transport: t,
|
|
}
|
|
}
|
|
|
|
func getServiceAccountClient(ctx context.Context, opt *Options, credentialsData []byte) (*http.Client, error) {
|
|
scopes := driveScopes(opt.Scope)
|
|
conf, err := google.JWTConfigFromJSON(credentialsData, scopes...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error processing credentials: %w", err)
|
|
}
|
|
if opt.Impersonate != "" {
|
|
conf.Subject = opt.Impersonate
|
|
}
|
|
ctxWithSpecialClient := oauthutil.Context(ctx, getClient(ctx, opt))
|
|
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
|
|
}
|
|
|
|
func createOAuthClient(ctx context.Context, opt *Options, name string, m configmap.Mapper) (*http.Client, error) {
|
|
var oAuthClient *http.Client
|
|
var err error
|
|
|
|
// try loading service account credentials from env variable, then from a file
|
|
if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
|
|
loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error opening service account credentials file: %w", err)
|
|
}
|
|
opt.ServiceAccountCredentials = string(loadedCreds)
|
|
}
|
|
if opt.ServiceAccountCredentials != "" {
|
|
oAuthClient, err = getServiceAccountClient(ctx, opt, []byte(opt.ServiceAccountCredentials))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create oauth client from service account: %w", err)
|
|
}
|
|
} else {
|
|
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create oauth client: %w", err)
|
|
}
|
|
}
|
|
|
|
return oAuthClient, nil
|
|
}
|
|
|
|
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
|
if !isPowerOfTwo(int64(cs)) {
|
|
return fmt.Errorf("%v isn't a power of two", cs)
|
|
}
|
|
if cs < minChunkSize {
|
|
return fmt.Errorf("%s is less than %s", cs, minChunkSize)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
|
err = checkUploadChunkSize(cs)
|
|
if err == nil {
|
|
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
|
}
|
|
return
|
|
}
|
|
|
|
func checkUploadCutoff(cs fs.SizeSuffix) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
|
err = checkUploadCutoff(cs)
|
|
if err == nil {
|
|
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
|
|
}
|
|
return
|
|
}
|
|
|
|
// newFs partially constructs Fs from the path
|
|
//
|
|
// It constructs a valid Fs but doesn't attempt to figure out whether
|
|
// it is a file or a directory.
|
|
func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, error) {
|
|
// Parse config into Options struct
|
|
opt := new(Options)
|
|
err := configstruct.Set(m, opt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
err = checkUploadCutoff(opt.UploadCutoff)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("drive: upload cutoff: %w", err)
|
|
}
|
|
err = checkUploadChunkSize(opt.ChunkSize)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("drive: chunk size: %w", err)
|
|
}
|
|
|
|
oAuthClient, err := createOAuthClient(ctx, opt, name, m)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("drive: failed when making oauth client: %w", err)
|
|
}
|
|
|
|
root, err := parseDrivePath(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ci := fs.GetConfig(ctx)
|
|
f := &Fs{
|
|
name: name,
|
|
root: root,
|
|
opt: *opt,
|
|
ci: ci,
|
|
pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
|
|
m: m,
|
|
grouping: listRGrouping,
|
|
listRmu: new(sync.Mutex),
|
|
listRempties: make(map[string]struct{}),
|
|
dirResourceKeys: new(sync.Map),
|
|
}
|
|
f.isTeamDrive = opt.TeamDriveID != ""
|
|
f.fileFields = f.getFileFields()
|
|
f.features = (&fs.Features{
|
|
DuplicateFiles: true,
|
|
ReadMimeType: true,
|
|
WriteMimeType: true,
|
|
CanHaveEmptyDirectories: true,
|
|
ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
|
|
}).Fill(ctx, f)
|
|
|
|
// Create a new authorized Drive client.
|
|
f.client = oAuthClient
|
|
f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't create Drive client: %w", err)
|
|
}
|
|
|
|
if f.opt.V2DownloadMinSize >= 0 {
|
|
f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't create Drive v2 client: %w", err)
|
|
}
|
|
}
|
|
|
|
return f, nil
|
|
}
|
|
|
|
// NewFs constructs an Fs from the path, container:path
|
|
func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, error) {
|
|
f, err := newFs(ctx, name, path, m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set the root folder ID
|
|
if f.opt.RootFolderID != "" {
|
|
// use root_folder ID if set
|
|
f.rootFolderID = f.opt.RootFolderID
|
|
} else if f.isTeamDrive {
|
|
// otherwise use team_drive if set
|
|
f.rootFolderID = f.opt.TeamDriveID
|
|
} else {
|
|
// otherwise look up the actual root ID
|
|
rootID, err := f.getRootID(ctx)
|
|
if err != nil {
|
|
var gerr *googleapi.Error
|
|
if errors.As(err, &gerr) && gerr.Code == 404 {
|
|
// 404 means that this scope does not have permission to get the
|
|
// root so just use "root"
|
|
rootID = "root"
|
|
} else {
|
|
return nil, err
|
|
}
|
|
}
|
|
f.rootFolderID = rootID
|
|
fs.Debugf(f, "'root_folder_id = %s' - save this in the config to speed up startup", rootID)
|
|
}
|
|
|
|
f.dirCache = dircache.New(f.root, f.rootFolderID, f)
|
|
|
|
// If resource key is set then cache it for the root folder id
|
|
if f.opt.ResourceKey != "" {
|
|
f.dirResourceKeys.Store(f.rootFolderID, f.opt.ResourceKey)
|
|
}
|
|
|
|
// Parse extensions
|
|
if f.opt.Extensions != "" {
|
|
if f.opt.ExportExtensions != defaultExportExtensions {
|
|
return nil, errors.New("only one of 'formats' and 'export_formats' can be specified")
|
|
}
|
|
f.opt.Extensions, f.opt.ExportExtensions = "", f.opt.Extensions
|
|
}
|
|
f.exportExtensions, _, err = parseExtensions(f.opt.ExportExtensions, defaultExportExtensions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
_, f.importMimeTypes, err = parseExtensions(f.opt.ImportExtensions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Find the current root
|
|
err = f.dirCache.FindRoot(ctx, false)
|
|
if err != nil {
|
|
// Assume it is a file
|
|
newRoot, remote := dircache.SplitPath(f.root)
|
|
tempF := *f
|
|
tempF.dirCache = dircache.New(newRoot, f.rootFolderID, &tempF)
|
|
tempF.root = newRoot
|
|
// Make new Fs which is the parent
|
|
err = tempF.dirCache.FindRoot(ctx, false)
|
|
if err != nil {
|
|
// No root so return old f
|
|
return f, nil
|
|
}
|
|
_, err := tempF.NewObject(ctx, remote)
|
|
if err != nil {
|
|
// unable to list folder so return old f
|
|
return f, nil
|
|
}
|
|
// XXX: update the old f here instead of returning tempF, since
|
|
// `features` were already filled with functions having *f as a receiver.
|
|
// See https://github.com/rclone/rclone/issues/2182
|
|
f.dirCache = tempF.dirCache
|
|
f.root = tempF.root
|
|
return f, fs.ErrorIsFile
|
|
}
|
|
// fmt.Printf("Root id %s", f.dirCache.RootID())
|
|
return f, nil
|
|
}
|
|
|
|
func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
|
|
modifiedDate := info.ModifiedTime
|
|
if f.opt.UseCreatedDate {
|
|
modifiedDate = info.CreatedTime
|
|
} else if f.opt.UseSharedDate && info.SharedWithMeTime != "" {
|
|
modifiedDate = info.SharedWithMeTime
|
|
}
|
|
size := info.Size
|
|
if f.opt.SizeAsQuota {
|
|
size = info.QuotaBytesUsed
|
|
}
|
|
return baseObject{
|
|
fs: f,
|
|
remote: remote,
|
|
id: info.Id,
|
|
modifiedDate: modifiedDate,
|
|
mimeType: info.MimeType,
|
|
bytes: size,
|
|
parents: info.Parents,
|
|
}
|
|
}
|
|
|
|
// getFileFields gets the fields for a normal file Get or List
|
|
func (f *Fs) getFileFields() (fields googleapi.Field) {
|
|
fields = partialFields
|
|
if f.opt.AuthOwnerOnly {
|
|
fields += ",owners"
|
|
}
|
|
if f.opt.UseSharedDate {
|
|
fields += ",sharedWithMeTime"
|
|
}
|
|
if f.opt.SkipChecksumGphotos {
|
|
fields += ",spaces"
|
|
}
|
|
if f.opt.SizeAsQuota {
|
|
fields += ",quotaBytesUsed"
|
|
}
|
|
return fields
|
|
}
|
|
|
|
// newRegularObject creates an fs.Object for a normal drive.File
|
|
func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
|
// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
|
|
if f.opt.SkipChecksumGphotos {
|
|
for _, space := range info.Spaces {
|
|
if space == "photos" {
|
|
info.Md5Checksum = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
o := &Object{
|
|
baseObject: f.newBaseObject(remote, info),
|
|
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
|
md5sum: strings.ToLower(info.Md5Checksum),
|
|
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
|
}
|
|
if info.ResourceKey != "" {
|
|
o.resourceKey = &info.ResourceKey
|
|
}
|
|
return o
|
|
}
|
|
|
|
// newDocumentObject creates an fs.Object for a google docs drive.File
|
|
func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
|
mediaType, _, err := mime.ParseMediaType(exportMimeType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
url := info.ExportLinks[mediaType]
|
|
baseObject := f.newBaseObject(remote+extension, info)
|
|
baseObject.bytes = -1
|
|
baseObject.mimeType = exportMimeType
|
|
return &documentObject{
|
|
baseObject: baseObject,
|
|
url: url,
|
|
documentMimeType: info.MimeType,
|
|
extLen: len(extension),
|
|
}, nil
|
|
}
|
|
|
|
// newLinkObject creates an fs.Object that represents a link a google docs drive.File
|
|
func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
|
t := linkTemplate(exportMimeType)
|
|
if t == nil {
|
|
return nil, fmt.Errorf("unsupported link type %s", exportMimeType)
|
|
}
|
|
xdgIcon := _mimeTypeToXDGLinkIcons[info.MimeType]
|
|
if xdgIcon == "" {
|
|
xdgIcon = defaultXDGIcon
|
|
}
|
|
var buf bytes.Buffer
|
|
err := t.Execute(&buf, struct {
|
|
URL, Title, XDGIcon string
|
|
}{
|
|
info.WebViewLink, info.Name, xdgIcon,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("executing template failed: %w", err)
|
|
}
|
|
|
|
baseObject := f.newBaseObject(remote+extension, info)
|
|
baseObject.bytes = int64(buf.Len())
|
|
baseObject.mimeType = exportMimeType
|
|
return &linkObject{
|
|
baseObject: baseObject,
|
|
content: buf.Bytes(),
|
|
extLen: len(extension),
|
|
}, nil
|
|
}
|
|
|
|
// newObjectWithInfo creates an fs.Object for any drive.File
|
|
//
|
|
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
|
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
|
|
// If item has MD5 sum it is a file stored on drive
|
|
if info.Md5Checksum != "" {
|
|
return f.newRegularObject(remote, info), nil
|
|
}
|
|
|
|
extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
|
|
return f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
|
|
}
|
|
|
|
// newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
|
|
//
|
|
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
|
func (f *Fs) newObjectWithExportInfo(
|
|
ctx context.Context, remote string, info *drive.File,
|
|
extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
|
|
// Note that resolveShortcut will have been called already if
|
|
// we are being called from a listing. However the drive.Item
|
|
// will have been resolved so this will do nothing.
|
|
info, err = f.resolveShortcut(ctx, info)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("new object: %w", err)
|
|
}
|
|
switch {
|
|
case info.MimeType == driveFolderType:
|
|
return nil, fs.ErrorIsDir
|
|
case info.MimeType == shortcutMimeType:
|
|
// We can only get here if f.opt.SkipShortcuts is set
|
|
// and not from a listing. This is unlikely.
|
|
fs.Debugf(remote, "Ignoring shortcut as skip shortcuts is set")
|
|
return nil, fs.ErrorObjectNotFound
|
|
case info.MimeType == shortcutMimeTypeDangling:
|
|
// Pretend a dangling shortcut is a regular object
|
|
// It will error if used, but appear in listings so it can be deleted
|
|
return f.newRegularObject(remote, info), nil
|
|
case info.Md5Checksum != "":
|
|
// If item has MD5 sum it is a file stored on drive
|
|
return f.newRegularObject(remote, info), nil
|
|
case f.opt.SkipGdocs:
|
|
fs.Debugf(remote, "Skipping google document type %q", info.MimeType)
|
|
return nil, fs.ErrorObjectNotFound
|
|
default:
|
|
// If item MimeType is in the ExportFormats then it is a google doc
|
|
if !isDocument {
|
|
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
|
|
return nil, fs.ErrorObjectNotFound
|
|
}
|
|
if extension == "" {
|
|
fs.Debugf(remote, "No export formats found for %q", info.MimeType)
|
|
return nil, fs.ErrorObjectNotFound
|
|
}
|
|
if isLinkMimeType(exportMimeType) {
|
|
return f.newLinkObject(remote, info, extension, exportMimeType)
|
|
}
|
|
return f.newDocumentObject(remote, info, extension, exportMimeType)
|
|
}
|
|
}
|
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
// it returns the error fs.ErrorObjectNotFound.
|
|
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
|
info, extension, exportName, exportMimeType, isDocument, err := f.getRemoteInfoWithExport(ctx, remote)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
remote = remote[:len(remote)-len(extension)]
|
|
obj, err := f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
|
|
switch {
|
|
case err != nil:
|
|
return nil, err
|
|
case obj == nil:
|
|
return nil, fs.ErrorObjectNotFound
|
|
default:
|
|
return obj, nil
|
|
}
|
|
}
|
|
|
|
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
|
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
|
// Find the leaf in pathID
|
|
pathID = actualID(pathID)
|
|
found, err = f.list(ctx, []string{pathID}, leaf, true, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
|
if !f.opt.SkipGdocs {
|
|
_, exportName, _, isDocument := f.findExportFormat(ctx, item)
|
|
if exportName == leaf {
|
|
pathIDOut = item.Id
|
|
return true
|
|
}
|
|
if isDocument {
|
|
return false
|
|
}
|
|
}
|
|
if item.Name == leaf {
|
|
pathIDOut = item.Id
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
return pathIDOut, found, err
|
|
}
|
|
|
|
// CreateDir makes a directory with pathID as parent and name leaf
|
|
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
|
leaf = f.opt.Enc.FromStandardName(leaf)
|
|
// fmt.Println("Making", path)
|
|
// Define the metadata for the directory we are going to create.
|
|
pathID = actualID(pathID)
|
|
createInfo := &drive.File{
|
|
Name: leaf,
|
|
Description: leaf,
|
|
MimeType: driveFolderType,
|
|
Parents: []string{pathID},
|
|
}
|
|
var info *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
info, err = f.svc.Files.Create(createInfo).
|
|
Fields("id").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return info.Id, nil
|
|
}
|
|
|
|
// isAuthOwned checks if any of the item owners is the authenticated owner
|
|
func isAuthOwned(item *drive.File) bool {
|
|
for _, owner := range item.Owners {
|
|
if owner.Me {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// linkTemplate returns the Template for a MIME type or nil if the
|
|
// MIME type does not represent a link
|
|
func linkTemplate(mt string) *template.Template {
|
|
templatesOnce.Do(func() {
|
|
_linkTemplates = map[string]*template.Template{
|
|
"application/x-link-desktop": template.Must(
|
|
template.New("application/x-link-desktop").Parse(desktopTemplate)),
|
|
"application/x-link-html": template.Must(
|
|
template.New("application/x-link-html").Parse(htmlTemplate)),
|
|
"application/x-link-url": template.Must(
|
|
template.New("application/x-link-url").Parse(urlTemplate)),
|
|
"application/x-link-webloc": template.Must(
|
|
template.New("application/x-link-webloc").Parse(weblocTemplate)),
|
|
}
|
|
})
|
|
return _linkTemplates[mt]
|
|
}
|
|
func (f *Fs) fetchFormats(ctx context.Context) {
|
|
fetchFormatsOnce.Do(func() {
|
|
var about *drive.About
|
|
var err error
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
about, err = f.svc.About.Get().
|
|
Fields("exportFormats,importFormats").
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
fs.Errorf(f, "Failed to get Drive exportFormats and importFormats: %v", err)
|
|
_exportFormats = map[string][]string{}
|
|
_importFormats = map[string][]string{}
|
|
return
|
|
}
|
|
_exportFormats = fixMimeTypeMap(about.ExportFormats)
|
|
_importFormats = fixMimeTypeMap(about.ImportFormats)
|
|
})
|
|
}
|
|
|
|
// exportFormats returns the export formats from drive, fetching them
|
|
// if necessary.
|
|
//
|
|
// if the fetch fails then it will not export any drive formats
|
|
func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
|
|
f.fetchFormats(ctx)
|
|
return _exportFormats
|
|
}
|
|
|
|
// importFormats returns the import formats from drive, fetching them
|
|
// if necessary.
|
|
//
|
|
// if the fetch fails then it will not import any drive formats
|
|
func (f *Fs) importFormats(ctx context.Context) map[string][]string {
|
|
f.fetchFormats(ctx)
|
|
return _importFormats
|
|
}
|
|
|
|
// findExportFormatByMimeType works out the optimum export settings
|
|
// for the given MIME type.
|
|
//
|
|
// Look through the exportExtensions and find the first format that can be
|
|
// converted. If none found then return ("", "", false)
|
|
func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
|
|
extension, mimeType string, isDocument bool) {
|
|
exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
|
|
if isDocument {
|
|
for _, _extension := range f.exportExtensions {
|
|
_mimeType := mime.TypeByExtension(_extension)
|
|
if isLinkMimeType(_mimeType) {
|
|
return _extension, _mimeType, true
|
|
}
|
|
for _, emt := range exportMimeTypes {
|
|
if emt == _mimeType {
|
|
return _extension, emt, true
|
|
}
|
|
if _mimeType == _mimeTypeCustomTransform[emt] {
|
|
return _extension, emt, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// If using a link type export and a more specific export
|
|
// hasn't been found all docs should be exported
|
|
for _, _extension := range f.exportExtensions {
|
|
_mimeType := mime.TypeByExtension(_extension)
|
|
if isLinkMimeType(_mimeType) {
|
|
return _extension, _mimeType, true
|
|
}
|
|
}
|
|
|
|
// else return empty
|
|
return "", "", isDocument
|
|
}
|
|
|
|
// findExportFormatByMimeType works out the optimum export settings
|
|
// for the given drive.File.
|
|
//
|
|
// Look through the exportExtensions and find the first format that can be
|
|
// converted. If none found then return ("", "", "", false)
|
|
func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension, filename, mimeType string, isDocument bool) {
|
|
// If item has MD5 sum it is a file stored on drive
|
|
if item.Md5Checksum != "" {
|
|
return
|
|
}
|
|
// Folders can't be documents
|
|
if item.MimeType == driveFolderType {
|
|
return
|
|
}
|
|
extension, mimeType, isDocument = f.findExportFormatByMimeType(ctx, item.MimeType)
|
|
if extension != "" {
|
|
filename = item.Name + extension
|
|
}
|
|
return
|
|
}
|
|
|
|
// findImportFormat finds the matching upload MIME type for a file
|
|
// If the given MIME type is in importMimeTypes, the matching upload
|
|
// MIME type is returned
|
|
//
|
|
// When no match is found "" is returned.
|
|
func (f *Fs) findImportFormat(ctx context.Context, mimeType string) string {
|
|
mimeType = fixMimeType(mimeType)
|
|
ifs := f.importFormats(ctx)
|
|
for _, mt := range f.importMimeTypes {
|
|
if mt == mimeType {
|
|
importMimeTypes := ifs[mimeType]
|
|
if l := len(importMimeTypes); l > 0 {
|
|
if l > 1 {
|
|
fs.Infof(f, "found %d import formats for %q: %q", l, mimeType, importMimeTypes)
|
|
}
|
|
return importMimeTypes[0]
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// List the objects and directories in dir into entries. The
|
|
// entries can be returned in any order but should be for a
|
|
// complete directory.
|
|
//
|
|
// dir should be "" to list the root, and should not have
|
|
// trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
|
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
directoryID = actualID(directoryID)
|
|
|
|
var iErr error
|
|
_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
|
entry, err := f.itemToDirEntry(ctx, path.Join(dir, item.Name), item)
|
|
if err != nil {
|
|
iErr = err
|
|
return true
|
|
}
|
|
if entry != nil {
|
|
entries = append(entries, entry)
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if iErr != nil {
|
|
return nil, iErr
|
|
}
|
|
// If listing the root of a teamdrive and got no entries,
|
|
// double check we have access
|
|
if f.isTeamDrive && len(entries) == 0 && f.root == "" && dir == "" {
|
|
err = f.teamDriveOK(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
// listREntry is a task to be executed by a litRRunner
|
|
type listREntry struct {
|
|
id, path string
|
|
}
|
|
|
|
// listRSlices is a helper struct to sort two slices at once
|
|
type listRSlices struct {
|
|
dirs []string
|
|
paths []string
|
|
}
|
|
|
|
func (s listRSlices) Sort() {
|
|
sort.Sort(s)
|
|
}
|
|
|
|
func (s listRSlices) Len() int {
|
|
return len(s.dirs)
|
|
}
|
|
|
|
func (s listRSlices) Swap(i, j int) {
|
|
s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
|
|
s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
|
|
}
|
|
|
|
func (s listRSlices) Less(i, j int) bool {
|
|
return s.dirs[i] < s.dirs[j]
|
|
}
|
|
|
|
// listRRunner will read dirIDs from the in channel, perform the file listing and call cb with each DirEntry.
|
|
//
|
|
// In each cycle it will read up to grouping entries from the in channel without blocking.
|
|
// If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
|
|
// nil is send to the out channel and the function returns.
|
|
func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error, sendJob func(listREntry)) {
|
|
var dirs []string
|
|
var paths []string
|
|
var grouping int32
|
|
|
|
for dir := range in {
|
|
dirs = append(dirs[:0], dir.id)
|
|
paths = append(paths[:0], dir.path)
|
|
grouping = atomic.LoadInt32(&f.grouping)
|
|
waitloop:
|
|
for i := int32(1); i < grouping; i++ {
|
|
select {
|
|
case d, ok := <-in:
|
|
if !ok {
|
|
break waitloop
|
|
}
|
|
dirs = append(dirs, d.id)
|
|
paths = append(paths, d.path)
|
|
default:
|
|
}
|
|
}
|
|
listRSlices{dirs, paths}.Sort()
|
|
var iErr error
|
|
foundItems := false
|
|
_, err := f.list(ctx, dirs, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
|
// shared with me items have no parents when at the root
|
|
if f.opt.SharedWithMe && len(item.Parents) == 0 && len(paths) == 1 && paths[0] == "" {
|
|
item.Parents = dirs
|
|
}
|
|
for _, parent := range item.Parents {
|
|
var i int
|
|
foundItems = true
|
|
earlyExit := false
|
|
// If only one item in paths then no need to search for the ID
|
|
// assuming google drive is doing its job properly.
|
|
//
|
|
// Note that we at the root when len(paths) == 1 && paths[0] == ""
|
|
if len(paths) == 1 {
|
|
// don't check parents at root because
|
|
// - shared with me items have no parents at the root
|
|
// - if using a root alias, e.g. "root" or "appDataFolder" the ID won't match
|
|
i = 0
|
|
// items at root can have more than one parent so we need to put
|
|
// the item in just once.
|
|
earlyExit = true
|
|
} else {
|
|
// only handle parents that are in the requested dirs list if not at root
|
|
i = sort.SearchStrings(dirs, parent)
|
|
if i == len(dirs) || dirs[i] != parent {
|
|
continue
|
|
}
|
|
}
|
|
remote := path.Join(paths[i], item.Name)
|
|
entry, err := f.itemToDirEntry(ctx, remote, item)
|
|
if err != nil {
|
|
iErr = err
|
|
return true
|
|
}
|
|
|
|
err = cb(entry)
|
|
if err != nil {
|
|
iErr = err
|
|
return true
|
|
}
|
|
|
|
// If didn't check parents then insert only once
|
|
if earlyExit {
|
|
break
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
// Found no items in more than one directory. Retry these as
|
|
// individual directories This is to work around a bug in google
|
|
// drive where (A in parents) or (B in parents) returns nothing
|
|
// sometimes. See #3114, #4289 and
|
|
// https://issuetracker.google.com/issues/149522397
|
|
if len(dirs) > 1 && !foundItems {
|
|
if atomic.SwapInt32(&f.grouping, 1) != 1 {
|
|
fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
|
|
}
|
|
f.listRmu.Lock()
|
|
for i := range dirs {
|
|
// Requeue the jobs
|
|
job := listREntry{id: dirs[i], path: paths[i]}
|
|
sendJob(job)
|
|
// Make a note of these dirs - if they all turn
|
|
// out to be empty then we can re-enable grouping
|
|
f.listRempties[dirs[i]] = struct{}{}
|
|
}
|
|
f.listRmu.Unlock()
|
|
fs.Debugf(f, "Recycled %d entries", len(dirs))
|
|
}
|
|
// If using a grouping of 1 and dir was empty then check to see if it
|
|
// is part of the group that caused grouping to be disabled.
|
|
if grouping == 1 && len(dirs) == 1 && !foundItems {
|
|
f.listRmu.Lock()
|
|
if _, found := f.listRempties[dirs[0]]; found {
|
|
// Remove the ID
|
|
delete(f.listRempties, dirs[0])
|
|
// If no empties left => all the directories that
|
|
// triggered the grouping being set to 1 were actually
|
|
// empty so must have made a mistake
|
|
if len(f.listRempties) == 0 {
|
|
if atomic.SwapInt32(&f.grouping, listRGrouping) != listRGrouping {
|
|
fs.Debugf(f, "Re-enabling ListR as previous detection was in error")
|
|
}
|
|
}
|
|
}
|
|
f.listRmu.Unlock()
|
|
}
|
|
|
|
for range dirs {
|
|
wg.Done()
|
|
}
|
|
|
|
if iErr != nil {
|
|
out <- iErr
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
out <- err
|
|
return
|
|
}
|
|
}
|
|
out <- nil
|
|
}
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
// from dir recursively into out.
|
|
//
|
|
// dir should be "" to start from the root, and should not
|
|
// have trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
//
|
|
// It should call callback for each tranche of entries read.
|
|
// These need not be returned in any particular order. If
|
|
// callback returns an error then the listing will stop
|
|
// immediately.
|
|
//
|
|
// Don't implement this unless you have a more efficient way
|
|
// of listing recursively that doing a directory traversal.
|
|
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
|
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
directoryID = actualID(directoryID)
|
|
|
|
mu := sync.Mutex{} // protects in and overflow
|
|
wg := sync.WaitGroup{}
|
|
in := make(chan listREntry, listRInputBuffer)
|
|
out := make(chan error, f.ci.Checkers)
|
|
list := walk.NewListRHelper(callback)
|
|
overflow := []listREntry{}
|
|
listed := 0
|
|
|
|
// Send a job to the input channel if not closed. If the job
|
|
// won't fit then queue it in the overflow slice.
|
|
//
|
|
// This will not block if the channel is full.
|
|
sendJob := func(job listREntry) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if in == nil {
|
|
return
|
|
}
|
|
wg.Add(1)
|
|
select {
|
|
case in <- job:
|
|
default:
|
|
overflow = append(overflow, job)
|
|
wg.Add(-1)
|
|
}
|
|
}
|
|
|
|
// Send the entry to the caller, queueing any directories as new jobs
|
|
cb := func(entry fs.DirEntry) error {
|
|
if d, isDir := entry.(*fs.Dir); isDir {
|
|
job := listREntry{actualID(d.ID()), d.Remote()}
|
|
sendJob(job)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
listed++
|
|
return list.Add(entry)
|
|
}
|
|
|
|
wg.Add(1)
|
|
in <- listREntry{directoryID, dir}
|
|
|
|
for i := 0; i < f.ci.Checkers; i++ {
|
|
go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
|
|
}
|
|
go func() {
|
|
// wait until the all directories are processed
|
|
wg.Wait()
|
|
// if the input channel overflowed add the collected entries to the channel now
|
|
for len(overflow) > 0 {
|
|
mu.Lock()
|
|
l := len(overflow)
|
|
// only fill half of the channel to prevent entries being put into overflow again
|
|
if l > listRInputBuffer/2 {
|
|
l = listRInputBuffer / 2
|
|
}
|
|
wg.Add(l)
|
|
for _, d := range overflow[:l] {
|
|
in <- d
|
|
}
|
|
overflow = overflow[l:]
|
|
mu.Unlock()
|
|
|
|
// wait again for the completion of all directories
|
|
wg.Wait()
|
|
}
|
|
mu.Lock()
|
|
if in != nil {
|
|
// notify all workers to exit
|
|
close(in)
|
|
in = nil
|
|
}
|
|
mu.Unlock()
|
|
}()
|
|
// wait until the all workers to finish
|
|
for i := 0; i < f.ci.Checkers; i++ {
|
|
e := <-out
|
|
mu.Lock()
|
|
// if one worker returns an error early, close the input so all other workers exit
|
|
if e != nil && in != nil {
|
|
err = e
|
|
close(in)
|
|
in = nil
|
|
}
|
|
mu.Unlock()
|
|
}
|
|
|
|
close(out)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = list.Flush()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// If listing the root of a teamdrive and got no entries,
|
|
// double check we have access
|
|
if f.isTeamDrive && listed == 0 && f.root == "" && dir == "" {
|
|
err = f.teamDriveOK(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
const shortcutSeparator = '\t'
|
|
|
|
// joinID adds an actual drive ID to the shortcut ID it came from
|
|
//
|
|
// directoryIDs in the dircache are these composite directory IDs so
|
|
// we must always unpack them before use.
|
|
func joinID(actual, shortcut string) string {
|
|
return actual + string(shortcutSeparator) + shortcut
|
|
}
|
|
|
|
// splitID separates an actual ID and a shortcut ID from a composite
|
|
// ID. If there was no shortcut ID then it will return "" for it.
|
|
func splitID(compositeID string) (actualID, shortcutID string) {
|
|
i := strings.IndexRune(compositeID, shortcutSeparator)
|
|
if i < 0 {
|
|
return compositeID, ""
|
|
}
|
|
return compositeID[:i], compositeID[i+1:]
|
|
}
|
|
|
|
// isShortcutID returns true if compositeID refers to a shortcut
|
|
func isShortcutID(compositeID string) bool {
|
|
return strings.ContainsRune(compositeID, shortcutSeparator)
|
|
}
|
|
|
|
// actualID returns an actual ID from a composite ID
|
|
func actualID(compositeID string) (actualID string) {
|
|
actualID, _ = splitID(compositeID)
|
|
return actualID
|
|
}
|
|
|
|
// shortcutID returns a shortcut ID from a composite ID if available,
|
|
// or the actual ID if not.
|
|
func shortcutID(compositeID string) (shortcutID string) {
|
|
actualID, shortcutID := splitID(compositeID)
|
|
if shortcutID != "" {
|
|
return shortcutID
|
|
}
|
|
return actualID
|
|
}
|
|
|
|
// isShortcut returns true of the item is a shortcut
|
|
func isShortcut(item *drive.File) bool {
|
|
return item.MimeType == shortcutMimeType && item.ShortcutDetails != nil
|
|
}
|
|
|
|
// Dereference shortcut if required. It returns the newItem (which may
|
|
// be just item).
|
|
//
|
|
// If we return a new item then the ID will be adjusted to be a
|
|
// composite of the actual ID and the shortcut ID. This is to make
|
|
// sure that we have decided in all use places what we are doing with
|
|
// the ID.
|
|
//
|
|
// Note that we assume shortcuts can't point to shortcuts. Google
|
|
// drive web interface doesn't offer the option to create a shortcut
|
|
// to a shortcut. The documentation is silent on the issue.
|
|
func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *drive.File, err error) {
|
|
if f.opt.SkipShortcuts || item.MimeType != shortcutMimeType {
|
|
return item, nil
|
|
}
|
|
if item.ShortcutDetails == nil {
|
|
fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
|
|
return item, nil
|
|
}
|
|
newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
|
|
if err != nil {
|
|
var gerr *googleapi.Error
|
|
if errors.As(err, &gerr) && gerr.Code == 404 {
|
|
// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
|
|
fs.Logf(nil, "Dangling shortcut %q detected", item.Name)
|
|
item.MimeType = shortcutMimeTypeDangling
|
|
return item, nil
|
|
}
|
|
return nil, fmt.Errorf("failed to resolve shortcut: %w", err)
|
|
}
|
|
// make sure we use the Name, Parents and Trashed from the original item
|
|
newItem.Name = item.Name
|
|
newItem.Parents = item.Parents
|
|
newItem.Trashed = item.Trashed
|
|
// the new ID is a composite ID
|
|
newItem.Id = joinID(newItem.Id, item.Id)
|
|
return newItem, nil
|
|
}
|
|
|
|
// itemToDirEntry converts a drive.File to an fs.DirEntry.
|
|
// When the drive.File cannot be represented as an fs.DirEntry
|
|
// (nil, nil) is returned.
|
|
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
|
switch {
|
|
case item.MimeType == driveFolderType:
|
|
// cache the directory ID for later lookups
|
|
f.dirCache.Put(remote, item.Id)
|
|
// cache the resource key for later lookups
|
|
if item.ResourceKey != "" {
|
|
f.dirResourceKeys.Store(item.Id, item.ResourceKey)
|
|
}
|
|
when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
|
|
d := fs.NewDir(remote, when).SetID(item.Id)
|
|
if len(item.Parents) > 0 {
|
|
d.SetParentID(item.Parents[0])
|
|
}
|
|
return d, nil
|
|
case f.opt.AuthOwnerOnly && !isAuthOwned(item):
|
|
// ignore object
|
|
default:
|
|
entry, err = f.newObjectWithInfo(ctx, remote, item)
|
|
if err == fs.ErrorObjectNotFound {
|
|
return nil, nil
|
|
}
|
|
return entry, err
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// Creates a drive.File info from the parameters passed in.
|
|
//
|
|
// Used to create new objects
|
|
func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Time) (*drive.File, error) {
|
|
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
directoryID = actualID(directoryID)
|
|
|
|
leaf = f.opt.Enc.FromStandardName(leaf)
|
|
// Define the metadata for the file we are going to create.
|
|
createInfo := &drive.File{
|
|
Name: leaf,
|
|
Description: leaf,
|
|
Parents: []string{directoryID},
|
|
ModifiedTime: modTime.Format(timeFormatOut),
|
|
}
|
|
return createInfo, nil
|
|
}
|
|
|
|
// Put the object
|
|
//
|
|
// Copy the reader in to the new object which is returned.
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
existingObj, err := f.NewObject(ctx, src.Remote())
|
|
switch err {
|
|
case nil:
|
|
return existingObj, existingObj.Update(ctx, in, src, options...)
|
|
case fs.ErrorObjectNotFound:
|
|
// Not found so create it
|
|
return f.PutUnchecked(ctx, in, src, options...)
|
|
default:
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
return f.Put(ctx, in, src, options...)
|
|
}
|
|
|
|
// PutUnchecked uploads the object
|
|
//
|
|
// This will create a duplicate if we upload a new file without
|
|
// checking to see if there is one already - use Put() for that.
|
|
func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
|
remote := src.Remote()
|
|
size := src.Size()
|
|
modTime := src.ModTime(ctx)
|
|
srcMimeType := fs.MimeTypeFromName(remote)
|
|
srcExt := path.Ext(remote)
|
|
exportExt := ""
|
|
importMimeType := ""
|
|
|
|
if f.importMimeTypes != nil && !f.opt.SkipGdocs {
|
|
importMimeType = f.findImportFormat(ctx, srcMimeType)
|
|
|
|
if isInternalMimeType(importMimeType) {
|
|
remote = remote[:len(remote)-len(srcExt)]
|
|
|
|
exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
|
|
if exportExt == "" {
|
|
return nil, fmt.Errorf("no export format found for %q", importMimeType)
|
|
}
|
|
if exportExt != srcExt && !f.opt.AllowImportNameChange {
|
|
return nil, fmt.Errorf("can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt)
|
|
}
|
|
}
|
|
}
|
|
|
|
createInfo, err := f.createFileInfo(ctx, remote, modTime)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if importMimeType != "" {
|
|
createInfo.MimeType = importMimeType
|
|
} else {
|
|
createInfo.MimeType = fs.MimeTypeFromName(remote)
|
|
}
|
|
|
|
var info *drive.File
|
|
if size >= 0 && size < int64(f.opt.UploadCutoff) {
|
|
// Make the API request to upload metadata and file data.
|
|
// Don't retry, return a retry error instead
|
|
err = f.pacer.CallNoRetry(func() (bool, error) {
|
|
info, err = f.svc.Files.Create(createInfo).
|
|
Media(in, googleapi.ContentType(srcMimeType), googleapi.ChunkSize(0)).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
KeepRevisionForever(f.opt.KeepRevisionForever).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
// Upload the file in chunks
|
|
info, err = f.Upload(ctx, in, size, srcMimeType, "", remote, createInfo)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return f.newObjectWithInfo(ctx, remote, info)
|
|
}
|
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
// in into the first one and rmdirs the other directories.
|
|
func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
|
if len(dirs) < 2 {
|
|
return nil
|
|
}
|
|
newDirs := dirs[:0]
|
|
for _, dir := range dirs {
|
|
if isShortcutID(dir.ID()) {
|
|
fs.Infof(dir, "skipping shortcut directory")
|
|
continue
|
|
}
|
|
newDirs = append(newDirs, dir)
|
|
}
|
|
dirs = newDirs
|
|
if len(dirs) < 2 {
|
|
return nil
|
|
}
|
|
dstDir := dirs[0]
|
|
for _, srcDir := range dirs[1:] {
|
|
// list the objects
|
|
infos := []*drive.File{}
|
|
_, err := f.list(ctx, []string{srcDir.ID()}, "", false, false, f.opt.TrashedOnly, true, func(info *drive.File) bool {
|
|
infos = append(infos, info)
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("MergeDirs list failed on %v: %w", srcDir, err)
|
|
}
|
|
// move them into place
|
|
for _, info := range infos {
|
|
fs.Infof(srcDir, "merging %q", info.Name)
|
|
// Move the file into the destination
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
_, err = f.svc.Files.Update(info.Id, nil).
|
|
RemoveParents(srcDir.ID()).
|
|
AddParents(dstDir.ID()).
|
|
Fields("").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("MergeDirs move failed on %q in %v: %w", info.Name, srcDir, err)
|
|
}
|
|
}
|
|
// rmdir (into trash) the now empty source directory
|
|
fs.Infof(srcDir, "removing empty directory")
|
|
err = f.delete(ctx, srcDir.ID(), true)
|
|
if err != nil {
|
|
return fmt.Errorf("MergeDirs move failed to rmdir %q: %w", srcDir, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Mkdir creates the container if it doesn't exist
|
|
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
|
_, err := f.dirCache.FindDir(ctx, dir, true)
|
|
return err
|
|
}
|
|
|
|
// delete a file or directory unconditionally by ID
|
|
func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
|
|
return f.pacer.Call(func() (bool, error) {
|
|
var err error
|
|
if useTrash {
|
|
info := drive.File{
|
|
Trashed: true,
|
|
}
|
|
_, err = f.svc.Files.Update(id, &info).
|
|
Fields("").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
} else {
|
|
err = f.svc.Files.Delete(id).
|
|
Fields("").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
}
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
}
|
|
|
|
// purgeCheck removes the dir directory, if check is set then it
|
|
// refuses to do so if it has anything in
|
|
func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
|
root := path.Join(f.root, dir)
|
|
dc := f.dirCache
|
|
directoryID, err := dc.FindDir(ctx, dir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
directoryID, shortcutID := splitID(directoryID)
|
|
// if directory is a shortcut remove it regardless
|
|
if shortcutID != "" {
|
|
return f.delete(ctx, shortcutID, f.opt.UseTrash)
|
|
}
|
|
var trashedFiles = false
|
|
if check {
|
|
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
|
|
if !item.Trashed {
|
|
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
|
|
return true
|
|
}
|
|
fs.Debugf(dir, "Rmdir: contains trashed file: %q", item.Name)
|
|
trashedFiles = true
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
return fmt.Errorf("directory not empty")
|
|
}
|
|
}
|
|
if root != "" {
|
|
// trash the directory if it had trashed files
|
|
// in or the user wants to trash, otherwise
|
|
// delete it.
|
|
err = f.delete(ctx, directoryID, trashedFiles || f.opt.UseTrash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else if check {
|
|
return errors.New("can't purge root directory")
|
|
}
|
|
f.dirCache.FlushDir(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Rmdir deletes a directory
|
|
//
|
|
// Returns an error if it isn't empty
|
|
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
|
return f.purgeCheck(ctx, dir, true)
|
|
}
|
|
|
|
// Precision of the object storage system
|
|
func (f *Fs) Precision() time.Duration {
|
|
return time.Millisecond
|
|
}
|
|
|
|
// Copy src to this remote using server-side copy operations.
|
|
//
|
|
// This is stored with the remote path given.
|
|
//
|
|
// It returns the destination Object and a possible error.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
|
var srcObj *baseObject
|
|
ext := ""
|
|
isDoc := false
|
|
switch src := src.(type) {
|
|
case *Object:
|
|
srcObj = &src.baseObject
|
|
case *documentObject:
|
|
srcObj, ext = &src.baseObject, src.ext()
|
|
isDoc = true
|
|
case *linkObject:
|
|
srcObj, ext = &src.baseObject, src.ext()
|
|
default:
|
|
fs.Debugf(src, "Can't copy - not same remote type")
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
|
|
// Look to see if there is an existing object before we remove
|
|
// the extension from the remote
|
|
existingObject, _ := f.NewObject(ctx, remote)
|
|
|
|
// Adjust the remote name to be without the extension if we
|
|
// are about to create a doc.
|
|
if ext != "" {
|
|
if !strings.HasSuffix(remote, ext) {
|
|
fs.Debugf(src, "Can't copy - not same document type")
|
|
return nil, fs.ErrorCantCopy
|
|
}
|
|
remote = remote[:len(remote)-len(ext)]
|
|
}
|
|
|
|
createInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if isDoc {
|
|
// preserve the description on copy for docs
|
|
info, err := f.getFile(ctx, actualID(srcObj.id), "description")
|
|
if err != nil {
|
|
fs.Errorf(srcObj, "Failed to read description for Google Doc: %v", err)
|
|
} else {
|
|
createInfo.Description = info.Description
|
|
}
|
|
} else {
|
|
// don't overwrite the description on copy for files
|
|
// this should work for docs but it doesn't - it is probably a bug in Google Drive
|
|
createInfo.Description = ""
|
|
}
|
|
|
|
// get the ID of the thing to copy
|
|
// copy the contents if CopyShortcutContent
|
|
// else copy the shortcut only
|
|
|
|
id := shortcutID(srcObj.id)
|
|
|
|
if f.opt.CopyShortcutContent {
|
|
id = actualID(srcObj.id)
|
|
}
|
|
|
|
var info *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
copy := f.svc.Files.Copy(id, createInfo).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
KeepRevisionForever(f.opt.KeepRevisionForever)
|
|
srcObj.addResourceKey(copy.Header())
|
|
info, err = copy.Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
newObject, err := f.newObjectWithInfo(ctx, remote, info)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Google docs aren't preserving their mod time after copy, so set them explicitly
|
|
// See: https://github.com/rclone/rclone/issues/4517
|
|
//
|
|
// FIXME remove this when google fixes the problem!
|
|
if isDoc {
|
|
// A short sleep is needed here in order to make the
|
|
// change effective, without it is is ignored. This is
|
|
// probably some eventual consistency nastiness.
|
|
sleepTime := 2 * time.Second
|
|
fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
|
|
time.Sleep(sleepTime)
|
|
err = newObject.SetModTime(ctx, src.ModTime(ctx))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if existingObject != nil {
|
|
err = existingObject.Remove(ctx)
|
|
if err != nil {
|
|
fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
|
|
}
|
|
}
|
|
return newObject, nil
|
|
}
|
|
|
|
// Purge deletes all the files and the container
|
|
//
|
|
// Optional interface: Only implement this if you have a way of
|
|
// deleting all the files quicker than just running Remove() on the
|
|
// result of List()
|
|
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
|
if f.opt.TrashedOnly {
|
|
return errors.New("can't purge with --drive-trashed-only, use delete if you want to selectively delete files")
|
|
}
|
|
return f.purgeCheck(ctx, dir, false)
|
|
}
|
|
|
|
type cleanupResult struct {
|
|
Errors int
|
|
}
|
|
|
|
func (r cleanupResult) Error() string {
|
|
return fmt.Sprintf("%d errors during cleanup - see log", r.Errors)
|
|
}
|
|
|
|
func (f *Fs) cleanupTeamDrive(ctx context.Context, dir string, directoryID string) (r cleanupResult, err error) {
|
|
_, err = f.list(ctx, []string{directoryID}, "", false, false, true, false, func(item *drive.File) bool {
|
|
remote := path.Join(dir, item.Name)
|
|
if item.ExplicitlyTrashed { // description is wrong - can also be set for folders - no need to recurse them
|
|
err := f.delete(ctx, item.Id, false)
|
|
if err != nil {
|
|
r.Errors++
|
|
fs.Errorf(remote, "%v", err)
|
|
}
|
|
return false
|
|
}
|
|
|
|
if item.MimeType == driveFolderType {
|
|
if !isShortcutID(item.Id) {
|
|
rNew, _ := f.cleanupTeamDrive(ctx, remote, item.Id)
|
|
r.Errors += rNew.Errors
|
|
}
|
|
return false
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
err = fmt.Errorf("failed to list directory: %w", err)
|
|
r.Errors++
|
|
fs.Errorf(dir, "%v", err)
|
|
}
|
|
if r.Errors != 0 {
|
|
return r, r
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// CleanUp empties the trash
|
|
func (f *Fs) CleanUp(ctx context.Context) error {
|
|
if f.isTeamDrive {
|
|
directoryID, err := f.dirCache.FindDir(ctx, "", false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
directoryID = actualID(directoryID)
|
|
_, err = f.cleanupTeamDrive(ctx, "", directoryID)
|
|
return err
|
|
}
|
|
err := f.pacer.Call(func() (bool, error) {
|
|
err := f.svc.Files.EmptyTrash().Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fs.Logf(f, "Note that emptying the trash happens in the background and can take some time.")
|
|
return nil
|
|
}
|
|
|
|
// teamDriveOK checks to see if we can access the team drive
|
|
func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
|
|
if !f.isTeamDrive {
|
|
return nil
|
|
}
|
|
var td *drive.Drive
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
td, err = f.svc.Drives.Get(f.opt.TeamDriveID).Fields("name,id,capabilities,createdTime,restrictions").Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get Shared Drive info: %w", err)
|
|
}
|
|
fs.Debugf(f, "read info from Shared Drive %q", td.Name)
|
|
return err
|
|
}
|
|
|
|
// About gets quota information
|
|
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
|
if f.isTeamDrive {
|
|
err := f.teamDriveOK(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Teamdrives don't appear to have a usage API so just return empty
|
|
return &fs.Usage{}, nil
|
|
}
|
|
var about *drive.About
|
|
var err error
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
about, err = f.svc.About.Get().Fields("storageQuota").Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get Drive storageQuota: %w", err)
|
|
}
|
|
q := about.StorageQuota
|
|
usage := &fs.Usage{
|
|
Used: fs.NewUsageValue(q.UsageInDrive), // bytes in use
|
|
Trashed: fs.NewUsageValue(q.UsageInDriveTrash), // bytes in trash
|
|
Other: fs.NewUsageValue(q.Usage - q.UsageInDrive), // other usage e.g. gmail in drive
|
|
}
|
|
if q.Limit > 0 {
|
|
usage.Total = fs.NewUsageValue(q.Limit) // quota of bytes that can be used
|
|
usage.Free = fs.NewUsageValue(q.Limit - q.Usage) // bytes which can be uploaded before reaching the quota
|
|
}
|
|
return usage, nil
|
|
}
|
|
|
|
// Move src to this remote using server-side move operations.
|
|
//
|
|
// This is stored with the remote path given.
|
|
//
|
|
// It returns the destination Object and a possible error.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
|
var srcObj *baseObject
|
|
ext := ""
|
|
switch src := src.(type) {
|
|
case *Object:
|
|
srcObj = &src.baseObject
|
|
case *documentObject:
|
|
srcObj, ext = &src.baseObject, src.ext()
|
|
case *linkObject:
|
|
srcObj, ext = &src.baseObject, src.ext()
|
|
default:
|
|
fs.Debugf(src, "Can't move - not same remote type")
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
|
|
if ext != "" {
|
|
if !strings.HasSuffix(remote, ext) {
|
|
fs.Debugf(src, "Can't move - not same document type")
|
|
return nil, fs.ErrorCantMove
|
|
}
|
|
remote = remote[:len(remote)-len(ext)]
|
|
}
|
|
|
|
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
srcParentID = actualID(srcParentID)
|
|
|
|
// Temporary Object under construction
|
|
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dstParents := strings.Join(dstInfo.Parents, ",")
|
|
dstInfo.Parents = nil
|
|
|
|
// Do the move
|
|
var info *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
info, err = f.svc.Files.Update(shortcutID(srcObj.id), dstInfo).
|
|
RemoveParents(srcParentID).
|
|
AddParents(dstParents).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return f.newObjectWithInfo(ctx, remote, info)
|
|
}
|
|
|
|
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
|
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
|
id, err := f.dirCache.FindDir(ctx, remote, false)
|
|
if err == nil {
|
|
fs.Debugf(f, "attempting to share directory '%s'", remote)
|
|
id = shortcutID(id)
|
|
} else {
|
|
fs.Debugf(f, "attempting to share single file '%s'", remote)
|
|
o, err := f.NewObject(ctx, remote)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
id = shortcutID(o.(fs.IDer).ID())
|
|
}
|
|
|
|
permission := &drive.Permission{
|
|
AllowFileDiscovery: false,
|
|
Role: "reader",
|
|
Type: "anyone",
|
|
}
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
// TODO: On TeamDrives this might fail if lacking permissions to change ACLs.
|
|
// Need to either check `canShare` attribute on the object or see if a sufficient permission is already present.
|
|
_, err = f.svc.Permissions.Create(id, permission).
|
|
Fields("").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return fmt.Sprintf("https://drive.google.com/open?id=%s", id), nil
|
|
}
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server-side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
|
srcFs, ok := src.(*Fs)
|
|
if !ok {
|
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
|
return fs.ErrorCantDirMove
|
|
}
|
|
|
|
srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_ = srcLeaf
|
|
|
|
dstDirectoryID = actualID(dstDirectoryID)
|
|
srcDirectoryID = actualID(srcDirectoryID)
|
|
|
|
// Do the move
|
|
patch := drive.File{
|
|
Name: dstLeaf,
|
|
}
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
_, err = f.svc.Files.Update(shortcutID(srcID), &patch).
|
|
RemoveParents(srcDirectoryID).
|
|
AddParents(dstDirectoryID).
|
|
Fields("").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
srcFs.dirCache.FlushDir(srcRemote)
|
|
return nil
|
|
}
|
|
|
|
// ChangeNotify calls the passed function with a path that has had changes.
|
|
// If the implementation uses polling, it should adhere to the given interval.
|
|
//
|
|
// Automatically restarts itself in case of unexpected behavior of the remote.
|
|
//
|
|
// Close the returned channel to stop being notified.
|
|
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
|
go func() {
|
|
// get the StartPageToken early so all changes from now on get processed
|
|
startPageToken, err := f.changeNotifyStartPageToken(ctx)
|
|
if err != nil {
|
|
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
|
}
|
|
var ticker *time.Ticker
|
|
var tickerC <-chan time.Time
|
|
for {
|
|
select {
|
|
case pollInterval, ok := <-pollIntervalChan:
|
|
if !ok {
|
|
if ticker != nil {
|
|
ticker.Stop()
|
|
}
|
|
return
|
|
}
|
|
if ticker != nil {
|
|
ticker.Stop()
|
|
ticker, tickerC = nil, nil
|
|
}
|
|
if pollInterval != 0 {
|
|
ticker = time.NewTicker(pollInterval)
|
|
tickerC = ticker.C
|
|
}
|
|
case <-tickerC:
|
|
if startPageToken == "" {
|
|
startPageToken, err = f.changeNotifyStartPageToken(ctx)
|
|
if err != nil {
|
|
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
|
continue
|
|
}
|
|
}
|
|
fs.Debugf(f, "Checking for changes on remote")
|
|
startPageToken, err = f.changeNotifyRunner(ctx, notifyFunc, startPageToken)
|
|
if err != nil {
|
|
fs.Infof(f, "Change notify listener failure: %s", err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
|
|
var startPageToken *drive.StartPageToken
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
changes := f.svc.Changes.GetStartPageToken().SupportsAllDrives(true)
|
|
if f.isTeamDrive {
|
|
changes.DriveId(f.opt.TeamDriveID)
|
|
}
|
|
startPageToken, err = changes.Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
return startPageToken.StartPageToken, nil
|
|
}
|
|
|
|
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startPageToken string) (newStartPageToken string, err error) {
|
|
pageToken := startPageToken
|
|
for {
|
|
var changeList *drive.ChangeList
|
|
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
changesCall := f.svc.Changes.List(pageToken).
|
|
Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))")
|
|
if f.opt.ListChunk > 0 {
|
|
changesCall.PageSize(f.opt.ListChunk)
|
|
}
|
|
changesCall.SupportsAllDrives(true)
|
|
changesCall.IncludeItemsFromAllDrives(true)
|
|
if f.isTeamDrive {
|
|
changesCall.DriveId(f.opt.TeamDriveID)
|
|
}
|
|
// If using appDataFolder then need to add Spaces
|
|
if f.rootFolderID == "appDataFolder" {
|
|
changesCall.Spaces("appDataFolder")
|
|
}
|
|
changeList, err = changesCall.Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
type entryType struct {
|
|
path string
|
|
entryType fs.EntryType
|
|
}
|
|
var pathsToClear []entryType
|
|
for _, change := range changeList.Changes {
|
|
// find the previous path
|
|
if path, ok := f.dirCache.GetInv(change.FileId); ok {
|
|
if change.File != nil && change.File.MimeType != driveFolderType {
|
|
pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})
|
|
} else {
|
|
pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryDirectory})
|
|
}
|
|
}
|
|
|
|
// find the new path
|
|
if change.File != nil {
|
|
change.File.Name = f.opt.Enc.ToStandardName(change.File.Name)
|
|
changeType := fs.EntryDirectory
|
|
if change.File.MimeType != driveFolderType {
|
|
changeType = fs.EntryObject
|
|
}
|
|
|
|
// translate the parent dir of this object
|
|
if len(change.File.Parents) > 0 {
|
|
for _, parent := range change.File.Parents {
|
|
if parentPath, ok := f.dirCache.GetInv(parent); ok {
|
|
// and append the drive file name to compute the full file name
|
|
newPath := path.Join(parentPath, change.File.Name)
|
|
// this will now clear the actual file too
|
|
pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
|
|
}
|
|
}
|
|
} else { // a true root object that is changed
|
|
pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
|
|
}
|
|
}
|
|
}
|
|
|
|
visitedPaths := make(map[string]struct{})
|
|
for _, entry := range pathsToClear {
|
|
if _, ok := visitedPaths[entry.path]; ok {
|
|
continue
|
|
}
|
|
visitedPaths[entry.path] = struct{}{}
|
|
notifyFunc(entry.path, entry.entryType)
|
|
}
|
|
|
|
switch {
|
|
case changeList.NewStartPageToken != "":
|
|
return changeList.NewStartPageToken, nil
|
|
case changeList.NextPageToken != "":
|
|
pageToken = changeList.NextPageToken
|
|
default:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing as an
|
|
// optional interface
|
|
func (f *Fs) DirCacheFlush() {
|
|
f.dirCache.ResetRoot()
|
|
}
|
|
|
|
// Hashes returns the supported hash sets.
|
|
func (f *Fs) Hashes() hash.Set {
|
|
return hash.Set(hash.MD5)
|
|
}
|
|
|
|
func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
|
|
chunkSizeInt, err := strconv.ParseInt(chunkSizeString, 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("couldn't convert chunk size to int: %w", err)
|
|
}
|
|
chunkSize := fs.SizeSuffix(chunkSizeInt)
|
|
if chunkSize == f.opt.ChunkSize {
|
|
return nil
|
|
}
|
|
err = checkUploadChunkSize(chunkSize)
|
|
if err == nil {
|
|
f.opt.ChunkSize = chunkSize
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (f *Fs) changeServiceAccountFile(ctx context.Context, file string) (err error) {
|
|
fs.Debugf(nil, "Changing Service Account File from %s to %s", f.opt.ServiceAccountFile, file)
|
|
if file == f.opt.ServiceAccountFile {
|
|
return nil
|
|
}
|
|
oldSvc := f.svc
|
|
oldv2Svc := f.v2Svc
|
|
oldOAuthClient := f.client
|
|
oldFile := f.opt.ServiceAccountFile
|
|
oldCredentials := f.opt.ServiceAccountCredentials
|
|
defer func() {
|
|
// Undo all the changes instead of doing selective undo's
|
|
if err != nil {
|
|
f.svc = oldSvc
|
|
f.v2Svc = oldv2Svc
|
|
f.client = oldOAuthClient
|
|
f.opt.ServiceAccountFile = oldFile
|
|
f.opt.ServiceAccountCredentials = oldCredentials
|
|
}
|
|
}()
|
|
f.opt.ServiceAccountFile = file
|
|
f.opt.ServiceAccountCredentials = ""
|
|
oAuthClient, err := createOAuthClient(ctx, &f.opt, f.name, f.m)
|
|
if err != nil {
|
|
return fmt.Errorf("drive: failed when making oauth client: %w", err)
|
|
}
|
|
f.client = oAuthClient
|
|
f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client))
|
|
if err != nil {
|
|
return fmt.Errorf("couldn't create Drive client: %w", err)
|
|
}
|
|
if f.opt.V2DownloadMinSize >= 0 {
|
|
f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client))
|
|
if err != nil {
|
|
return fmt.Errorf("couldn't create Drive v2 client: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Create a shortcut from (f, srcPath) to (dstFs, dstPath)
|
|
//
|
|
// Will not overwrite existing files
|
|
func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPath string) (o fs.Object, err error) {
|
|
srcFs := f
|
|
srcPath = strings.Trim(srcPath, "/")
|
|
dstPath = strings.Trim(dstPath, "/")
|
|
if dstPath == "" {
|
|
return nil, errors.New("shortcut destination can't be root directory")
|
|
}
|
|
|
|
// Find source
|
|
var srcID string
|
|
isDir := false
|
|
if srcPath == "" {
|
|
// source is root directory
|
|
srcID, err = f.dirCache.RootID(ctx, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
isDir = true
|
|
} else if srcObj, err := srcFs.NewObject(ctx, srcPath); err != nil {
|
|
if err != fs.ErrorIsDir {
|
|
return nil, fmt.Errorf("can't find source: %w", err)
|
|
}
|
|
// source was a directory
|
|
srcID, err = srcFs.dirCache.FindDir(ctx, srcPath, false)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to find source dir: %w", err)
|
|
}
|
|
isDir = true
|
|
} else {
|
|
// source was a file
|
|
srcID = srcObj.(*Object).id
|
|
}
|
|
srcID = actualID(srcID) // link to underlying object not to shortcut
|
|
|
|
// Find destination
|
|
_, err = dstFs.NewObject(ctx, dstPath)
|
|
if err != fs.ErrorObjectNotFound {
|
|
if err == nil {
|
|
err = errors.New("existing file")
|
|
} else if err == fs.ErrorIsDir {
|
|
err = errors.New("existing directory")
|
|
}
|
|
return nil, fmt.Errorf("not overwriting shortcut target: %w", err)
|
|
}
|
|
|
|
// Create destination shortcut
|
|
createInfo, err := dstFs.createFileInfo(ctx, dstPath, time.Now())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("shortcut destination failed: %w", err)
|
|
}
|
|
createInfo.MimeType = shortcutMimeType
|
|
createInfo.ShortcutDetails = &drive.FileShortcutDetails{
|
|
TargetId: srcID,
|
|
}
|
|
|
|
var info *drive.File
|
|
err = dstFs.pacer.Call(func() (bool, error) {
|
|
info, err = dstFs.svc.Files.Create(createInfo).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
KeepRevisionForever(dstFs.opt.KeepRevisionForever).
|
|
Context(ctx).Do()
|
|
return dstFs.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("shortcut creation failed: %w", err)
|
|
}
|
|
if isDir {
|
|
return nil, nil
|
|
}
|
|
return dstFs.newObjectWithInfo(ctx, dstPath, info)
|
|
}
|
|
|
|
// List all team drives
|
|
func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.Drive, err error) {
|
|
drives = []*drive.Drive{}
|
|
listTeamDrives := f.svc.Drives.List().PageSize(100)
|
|
var defaultFs Fs // default Fs with default Options
|
|
for {
|
|
var teamDrives *drive.DriveList
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
teamDrives, err = listTeamDrives.Context(ctx).Do()
|
|
return defaultFs.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return drives, fmt.Errorf("listing Team Drives failed: %w", err)
|
|
}
|
|
drives = append(drives, teamDrives.Drives...)
|
|
if teamDrives.NextPageToken == "" {
|
|
break
|
|
}
|
|
listTeamDrives.PageToken(teamDrives.NextPageToken)
|
|
}
|
|
return drives, nil
|
|
}
|
|
|
|
type unTrashResult struct {
|
|
Untrashed int
|
|
Errors int
|
|
}
|
|
|
|
func (r unTrashResult) Error() string {
|
|
return fmt.Sprintf("%d errors while untrashing - see log", r.Errors)
|
|
}
|
|
|
|
// Restore the trashed files from dir, directoryID recursing if needed
|
|
func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurse bool) (r unTrashResult, err error) {
|
|
directoryID = actualID(directoryID)
|
|
fs.Debugf(dir, "finding trash to restore in directory %q", directoryID)
|
|
_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
|
|
remote := path.Join(dir, item.Name)
|
|
if item.ExplicitlyTrashed {
|
|
fs.Infof(remote, "restoring %q", item.Id)
|
|
if operations.SkipDestructive(ctx, remote, "restore") {
|
|
return false
|
|
}
|
|
update := drive.File{
|
|
ForceSendFields: []string{"Trashed"}, // necessary to set false value
|
|
Trashed: false,
|
|
}
|
|
err := f.pacer.Call(func() (bool, error) {
|
|
_, err := f.svc.Files.Update(item.Id, &update).
|
|
SupportsAllDrives(true).
|
|
Fields("trashed").
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
err = fmt.Errorf("failed to restore: %w", err)
|
|
r.Errors++
|
|
fs.Errorf(remote, "%v", err)
|
|
} else {
|
|
r.Untrashed++
|
|
}
|
|
}
|
|
if recurse && item.MimeType == "application/vnd.google-apps.folder" {
|
|
if !isShortcutID(item.Id) {
|
|
rNew, _ := f.unTrash(ctx, remote, item.Id, recurse)
|
|
r.Untrashed += rNew.Untrashed
|
|
r.Errors += rNew.Errors
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
err = fmt.Errorf("failed to list directory: %w", err)
|
|
r.Errors++
|
|
fs.Errorf(dir, "%v", err)
|
|
}
|
|
if r.Errors != 0 {
|
|
return r, r
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// Untrash dir
|
|
func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTrashResult, err error) {
|
|
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
|
if err != nil {
|
|
r.Errors++
|
|
return r, err
|
|
}
|
|
return f.unTrash(ctx, dir, directoryID, true)
|
|
}
|
|
|
|
// copy file with id to dest
|
|
func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
|
|
info, err := f.getFile(ctx, id, f.fileFields)
|
|
if err != nil {
|
|
return fmt.Errorf("couldn't find id: %w", err)
|
|
}
|
|
if info.MimeType == driveFolderType {
|
|
return fmt.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
|
|
}
|
|
info.Name = f.opt.Enc.ToStandardName(info.Name)
|
|
o, err := f.newObjectWithInfo(ctx, info.Name, info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
destDir, destLeaf, err := fspath.Split(dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if destLeaf == "" {
|
|
destLeaf = path.Base(o.Remote())
|
|
}
|
|
if destDir == "" {
|
|
destDir = "."
|
|
}
|
|
dstFs, err := cache.Get(ctx, destDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = operations.Copy(ctx, dstFs, nil, destLeaf, o)
|
|
if err != nil {
|
|
return fmt.Errorf("copy failed: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var commandHelp = []fs.CommandHelp{{
|
|
Name: "get",
|
|
Short: "Get command for fetching the drive config parameters",
|
|
Long: `This is a get command which will be used to fetch the various drive config parameters
|
|
|
|
Usage Examples:
|
|
|
|
rclone backend get drive: [-o service_account_file] [-o chunk_size]
|
|
rclone rc backend/command command=get fs=drive: [-o service_account_file] [-o chunk_size]
|
|
`,
|
|
Opts: map[string]string{
|
|
"chunk_size": "show the current upload chunk size",
|
|
"service_account_file": "show the current service account file",
|
|
},
|
|
}, {
|
|
Name: "set",
|
|
Short: "Set command for updating the drive config parameters",
|
|
Long: `This is a set command which will be used to update the various drive config parameters
|
|
|
|
Usage Examples:
|
|
|
|
rclone backend set drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
|
rclone rc backend/command command=set fs=drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
|
`,
|
|
Opts: map[string]string{
|
|
"chunk_size": "update the current upload chunk size",
|
|
"service_account_file": "update the current service account file",
|
|
},
|
|
}, {
|
|
Name: "shortcut",
|
|
Short: "Create shortcuts from files or directories",
|
|
Long: `This command creates shortcuts from files or directories.
|
|
|
|
Usage:
|
|
|
|
rclone backend shortcut drive: source_item destination_shortcut
|
|
rclone backend shortcut drive: source_item -o target=drive2: destination_shortcut
|
|
|
|
In the first example this creates a shortcut from the "source_item"
|
|
which can be a file or a directory to the "destination_shortcut". The
|
|
"source_item" and the "destination_shortcut" should be relative paths
|
|
from "drive:"
|
|
|
|
In the second example this creates a shortcut from the "source_item"
|
|
relative to "drive:" to the "destination_shortcut" relative to
|
|
"drive2:". This may fail with a permission error if the user
|
|
authenticated with "drive2:" can't read files from "drive:".
|
|
`,
|
|
Opts: map[string]string{
|
|
"target": "optional target remote for the shortcut destination",
|
|
},
|
|
}, {
|
|
Name: "drives",
|
|
Short: "List the Shared Drives available to this account",
|
|
Long: `This command lists the Shared Drives (Team Drives) available to this
|
|
account.
|
|
|
|
Usage:
|
|
|
|
rclone backend [-o config] drives drive:
|
|
|
|
This will return a JSON list of objects like this
|
|
|
|
[
|
|
{
|
|
"id": "0ABCDEF-01234567890",
|
|
"kind": "drive#teamDrive",
|
|
"name": "My Drive"
|
|
},
|
|
{
|
|
"id": "0ABCDEFabcdefghijkl",
|
|
"kind": "drive#teamDrive",
|
|
"name": "Test Drive"
|
|
}
|
|
]
|
|
|
|
With the -o config parameter it will output the list in a format
|
|
suitable for adding to a config file to make aliases for all the
|
|
drives found and a combined drive.
|
|
|
|
[My Drive]
|
|
type = alias
|
|
remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=:
|
|
|
|
[Test Drive]
|
|
type = alias
|
|
remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
|
|
|
|
[AllDrives]
|
|
type = combine
|
|
upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
|
|
|
|
Adding this to the rclone config file will cause those team drives to
|
|
be accessible with the aliases shown. Any illegal characters will be
|
|
substituted with "_" and duplicate names will have numbers suffixed.
|
|
It will also add a remote called AllDrives which shows all the shared
|
|
drives combined into one directory tree.
|
|
`,
|
|
}, {
|
|
Name: "untrash",
|
|
Short: "Untrash files and directories",
|
|
Long: `This command untrashes all the files and directories in the directory
|
|
passed in recursively.
|
|
|
|
Usage:
|
|
|
|
This takes an optional directory to trash which make this easier to
|
|
use via the API.
|
|
|
|
rclone backend untrash drive:directory
|
|
rclone backend -i untrash drive:directory subdir
|
|
|
|
Use the -i flag to see what would be restored before restoring it.
|
|
|
|
Result:
|
|
|
|
{
|
|
"Untrashed": 17,
|
|
"Errors": 0
|
|
}
|
|
`,
|
|
}, {
|
|
Name: "copyid",
|
|
Short: "Copy files by ID",
|
|
Long: `This command copies files by ID
|
|
|
|
Usage:
|
|
|
|
rclone backend copyid drive: ID path
|
|
rclone backend copyid drive: ID1 path1 ID2 path2
|
|
|
|
It copies the drive file with ID given to the path (an rclone path which
|
|
will be passed internally to rclone copyto). The ID and path pairs can be
|
|
repeated.
|
|
|
|
The path should end with a / to indicate copy the file as named to
|
|
this directory. If it doesn't end with a / then the last path
|
|
component will be used as the file name.
|
|
|
|
If the destination is a drive backend then server-side copying will be
|
|
attempted if possible.
|
|
|
|
Use the -i flag to see what would be copied before copying.
|
|
`,
|
|
}, {
|
|
Name: "exportformats",
|
|
Short: "Dump the export formats for debug purposes",
|
|
}, {
|
|
Name: "importformats",
|
|
Short: "Dump the import formats for debug purposes",
|
|
}}
|
|
|
|
// Command the backend to run a named command
|
|
//
|
|
// The command run is name
|
|
// args may be used to read arguments from
|
|
// opts may be used to read optional arguments from
|
|
//
|
|
// The result should be capable of being JSON encoded
|
|
// If it is a string or a []string it will be shown to the user
|
|
// otherwise it will be JSON encoded and shown to the user like that
|
|
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
|
|
switch name {
|
|
case "get":
|
|
out := make(map[string]string)
|
|
if _, ok := opt["service_account_file"]; ok {
|
|
out["service_account_file"] = f.opt.ServiceAccountFile
|
|
}
|
|
if _, ok := opt["chunk_size"]; ok {
|
|
out["chunk_size"] = f.opt.ChunkSize.String()
|
|
}
|
|
return out, nil
|
|
case "set":
|
|
out := make(map[string]map[string]string)
|
|
if serviceAccountFile, ok := opt["service_account_file"]; ok {
|
|
serviceAccountMap := make(map[string]string)
|
|
serviceAccountMap["previous"] = f.opt.ServiceAccountFile
|
|
if err = f.changeServiceAccountFile(ctx, serviceAccountFile); err != nil {
|
|
return out, err
|
|
}
|
|
f.m.Set("service_account_file", serviceAccountFile)
|
|
serviceAccountMap["current"] = f.opt.ServiceAccountFile
|
|
out["service_account_file"] = serviceAccountMap
|
|
}
|
|
if chunkSize, ok := opt["chunk_size"]; ok {
|
|
chunkSizeMap := make(map[string]string)
|
|
chunkSizeMap["previous"] = f.opt.ChunkSize.String()
|
|
if err = f.changeChunkSize(chunkSize); err != nil {
|
|
return out, err
|
|
}
|
|
chunkSizeString := f.opt.ChunkSize.String()
|
|
f.m.Set("chunk_size", chunkSizeString)
|
|
chunkSizeMap["current"] = chunkSizeString
|
|
out["chunk_size"] = chunkSizeMap
|
|
}
|
|
return out, nil
|
|
case "shortcut":
|
|
if len(arg) != 2 {
|
|
return nil, errors.New("need exactly 2 arguments")
|
|
}
|
|
dstFs := f
|
|
target, ok := opt["target"]
|
|
if ok {
|
|
targetFs, err := cache.Get(ctx, target)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't find target: %w", err)
|
|
}
|
|
dstFs, ok = targetFs.(*Fs)
|
|
if !ok {
|
|
return nil, errors.New("target is not a drive backend")
|
|
}
|
|
}
|
|
return f.makeShortcut(ctx, arg[0], dstFs, arg[1])
|
|
case "drives":
|
|
drives, err := f.listTeamDrives(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
re := regexp.MustCompile(`[^\w_. -]+`)
|
|
if _, ok := opt["config"]; ok {
|
|
lines := []string{}
|
|
upstreams := []string{}
|
|
names := make(map[string]struct{}, len(drives))
|
|
for i, drive := range drives {
|
|
name := re.ReplaceAllString(drive.Name, "_")
|
|
for {
|
|
if _, found := names[name]; !found {
|
|
break
|
|
}
|
|
name += fmt.Sprintf("-%d", i)
|
|
}
|
|
names[name] = struct{}{}
|
|
lines = append(lines, "")
|
|
lines = append(lines, fmt.Sprintf("[%s]", name))
|
|
lines = append(lines, "type = alias")
|
|
lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id))
|
|
upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name))
|
|
}
|
|
lines = append(lines, "")
|
|
lines = append(lines, "[AllDrives]")
|
|
lines = append(lines, "type = combine")
|
|
lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " ")))
|
|
return lines, nil
|
|
}
|
|
return drives, nil
|
|
case "untrash":
|
|
dir := ""
|
|
if len(arg) > 0 {
|
|
dir = arg[0]
|
|
}
|
|
return f.unTrashDir(ctx, dir, true)
|
|
case "copyid":
|
|
if len(arg)%2 != 0 {
|
|
return nil, errors.New("need an even number of arguments")
|
|
}
|
|
for len(arg) > 0 {
|
|
id, dest := arg[0], arg[1]
|
|
arg = arg[2:]
|
|
err = f.copyID(ctx, id, dest)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed copying %q to %q: %w", id, dest, err)
|
|
}
|
|
}
|
|
return nil, nil
|
|
case "exportformats":
|
|
return f.exportFormats(ctx), nil
|
|
case "importformats":
|
|
return f.importFormats(ctx), nil
|
|
default:
|
|
return nil, fs.ErrorCommandNotFound
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------
|
|
|
|
// Fs returns the parent Fs
|
|
func (o *baseObject) Fs() fs.Info {
|
|
return o.fs
|
|
}
|
|
|
|
// Return a string version
|
|
func (o *baseObject) String() string {
|
|
return o.remote
|
|
}
|
|
|
|
// Return a string version
|
|
func (o *Object) String() string {
|
|
if o == nil {
|
|
return "<nil>"
|
|
}
|
|
return o.remote
|
|
}
|
|
|
|
// Remote returns the remote path
|
|
func (o *baseObject) Remote() string {
|
|
return o.remote
|
|
}
|
|
|
|
// Hash returns the Md5sum of an object returning a lowercase hex string
|
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
|
if t != hash.MD5 {
|
|
return "", hash.ErrUnsupported
|
|
}
|
|
return o.md5sum, nil
|
|
}
|
|
func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
|
|
if t != hash.MD5 {
|
|
return "", hash.ErrUnsupported
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
// Size returns the size of an object in bytes
|
|
func (o *baseObject) Size() int64 {
|
|
return o.bytes
|
|
}
|
|
|
|
// getRemoteInfoWithExport returns a drive.File and the export settings for the remote
|
|
func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
|
|
info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
|
|
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
|
if err != nil {
|
|
if err == fs.ErrorDirNotFound {
|
|
return nil, "", "", "", false, fs.ErrorObjectNotFound
|
|
}
|
|
return nil, "", "", "", false, err
|
|
}
|
|
directoryID = actualID(directoryID)
|
|
|
|
found, err := f.list(ctx, []string{directoryID}, leaf, false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
|
if !f.opt.SkipGdocs {
|
|
extension, exportName, exportMimeType, isDocument = f.findExportFormat(ctx, item)
|
|
if exportName == leaf {
|
|
info = item
|
|
return true
|
|
}
|
|
if isDocument {
|
|
return false
|
|
}
|
|
}
|
|
if item.Name == leaf {
|
|
info = item
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
if err != nil {
|
|
return nil, "", "", "", false, err
|
|
}
|
|
if !found {
|
|
return nil, "", "", "", false, fs.ErrorObjectNotFound
|
|
}
|
|
return
|
|
}
|
|
|
|
// ModTime returns the modification time of the object
|
|
//
|
|
// It attempts to read the objects mtime and if that isn't present the
|
|
// LastModified returned in the http headers
|
|
func (o *baseObject) ModTime(ctx context.Context) time.Time {
|
|
modTime, err := time.Parse(timeFormatIn, o.modifiedDate)
|
|
if err != nil {
|
|
fs.Debugf(o, "Failed to read mtime from object: %v", err)
|
|
return time.Now()
|
|
}
|
|
return modTime
|
|
}
|
|
|
|
// SetModTime sets the modification time of the drive fs object
|
|
func (o *baseObject) SetModTime(ctx context.Context, modTime time.Time) error {
|
|
// New metadata
|
|
updateInfo := &drive.File{
|
|
ModifiedTime: modTime.Format(timeFormatOut),
|
|
}
|
|
// Set modified date
|
|
var info *drive.File
|
|
err := o.fs.pacer.Call(func() (bool, error) {
|
|
var err error
|
|
info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return o.fs.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Update info from read data
|
|
o.modifiedDate = info.ModifiedTime
|
|
return nil
|
|
}
|
|
|
|
// Storable returns a boolean as to whether this object is storable
|
|
func (o *baseObject) Storable() bool {
|
|
return true
|
|
}
|
|
|
|
// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
|
|
// object if required.
|
|
func (o *baseObject) addResourceKey(header http.Header) {
|
|
if o.resourceKey != nil {
|
|
header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
|
|
}
|
|
}
|
|
|
|
// httpResponse gets an http.Response object for the object
|
|
// using the url and method passed in
|
|
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
|
|
if url == "" {
|
|
return nil, nil, errors.New("forbidden to download - check sharing permission")
|
|
}
|
|
req, err = http.NewRequestWithContext(ctx, method, url, nil)
|
|
if err != nil {
|
|
return req, nil, err
|
|
}
|
|
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
|
if o.bytes == 0 {
|
|
// Don't supply range requests for 0 length objects as they always fail
|
|
delete(req.Header, "Range")
|
|
}
|
|
o.addResourceKey(req.Header)
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
res, err = o.fs.client.Do(req)
|
|
if err == nil {
|
|
err = googleapi.CheckResponse(res)
|
|
if err != nil {
|
|
_ = res.Body.Close() // ignore error
|
|
}
|
|
}
|
|
return o.fs.shouldRetry(ctx, err)
|
|
})
|
|
if err != nil {
|
|
return req, nil, err
|
|
}
|
|
return req, res, nil
|
|
}
|
|
|
|
// openDocumentFile represents a documentObject open for reading.
|
|
// Updates the object size after read successfully.
|
|
type openDocumentFile struct {
|
|
o *documentObject // Object we are reading for
|
|
in io.ReadCloser // reading from here
|
|
bytes int64 // number of bytes read on this connection
|
|
eof bool // whether we have read end of file
|
|
errored bool // whether we have encountered an error during reading
|
|
}
|
|
|
|
// Read bytes from the object - see io.Reader
|
|
func (file *openDocumentFile) Read(p []byte) (n int, err error) {
|
|
n, err = file.in.Read(p)
|
|
file.bytes += int64(n)
|
|
if err != nil && err != io.EOF {
|
|
file.errored = true
|
|
}
|
|
if err == io.EOF {
|
|
file.eof = true
|
|
}
|
|
return
|
|
}
|
|
|
|
// Close the object and update bytes read
|
|
func (file *openDocumentFile) Close() (err error) {
|
|
// If end of file, update bytes read
|
|
if file.eof && !file.errored {
|
|
fs.Debugf(file.o, "Updating size of doc after download to %v", file.bytes)
|
|
file.o.bytes = file.bytes
|
|
}
|
|
return file.in.Close()
|
|
}
|
|
|
|
// Check it satisfies the interfaces
|
|
var _ io.ReadCloser = (*openDocumentFile)(nil)
|
|
|
|
// Checks to see if err is a googleapi.Error with of type what
|
|
func isGoogleError(err error, what string) bool {
|
|
if gerr, ok := err.(*googleapi.Error); ok {
|
|
for _, error := range gerr.Errors {
|
|
if error.Reason == what {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// open a url for reading
|
|
func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
_, res, err := o.httpResponse(ctx, url, "GET", options)
|
|
if err != nil {
|
|
if isGoogleError(err, "cannotDownloadAbusiveFile") {
|
|
if o.fs.opt.AcknowledgeAbuse {
|
|
// Retry acknowledging abuse
|
|
if strings.ContainsRune(url, '?') {
|
|
url += "&"
|
|
} else {
|
|
url += "?"
|
|
}
|
|
url += "acknowledgeAbuse=true"
|
|
_, res, err = o.httpResponse(ctx, url, "GET", options)
|
|
} else {
|
|
err = fmt.Errorf("use the --drive-acknowledge-abuse flag to download this file: %w", err)
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open file failed: %w", err)
|
|
}
|
|
}
|
|
return res.Body, nil
|
|
}
|
|
|
|
// Open an object for read
|
|
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
if o.mimeType == shortcutMimeTypeDangling {
|
|
return nil, errors.New("can't read dangling shortcut")
|
|
}
|
|
if o.v2Download {
|
|
var v2File *drive_v2.File
|
|
err = o.fs.pacer.Call(func() (bool, error) {
|
|
v2File, err = o.fs.v2Svc.Files.Get(actualID(o.id)).
|
|
Fields("downloadUrl").
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return o.fs.shouldRetry(ctx, err)
|
|
})
|
|
if err == nil {
|
|
fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
|
|
o.url = v2File.DownloadUrl
|
|
o.v2Download = false
|
|
}
|
|
}
|
|
return o.baseObject.open(ctx, o.url, options...)
|
|
}
|
|
func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
// Update the size with what we are reading as it can change from
|
|
// the HEAD in the listing to this GET. This stops rclone marking
|
|
// the transfer as corrupted.
|
|
var offset, end int64 = 0, -1
|
|
var newOptions = options[:0]
|
|
for _, o := range options {
|
|
// Note that Range requests don't work on Google docs:
|
|
// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
|
|
// So do a subset of them manually
|
|
switch x := o.(type) {
|
|
case *fs.RangeOption:
|
|
offset, end = x.Start, x.End
|
|
case *fs.SeekOption:
|
|
offset, end = x.Offset, -1
|
|
default:
|
|
newOptions = append(newOptions, o)
|
|
}
|
|
}
|
|
options = newOptions
|
|
if offset != 0 {
|
|
return nil, errors.New("partial downloads are not supported while exporting Google Documents")
|
|
}
|
|
in, err = o.baseObject.open(ctx, o.url, options...)
|
|
if in != nil {
|
|
in = &openDocumentFile{o: o, in: in}
|
|
}
|
|
if end >= 0 {
|
|
in = readers.NewLimitedReadCloser(in, end-offset+1)
|
|
}
|
|
return
|
|
}
|
|
func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
|
var offset, limit int64 = 0, -1
|
|
var data = o.content
|
|
for _, option := range options {
|
|
switch x := option.(type) {
|
|
case *fs.SeekOption:
|
|
offset = x.Offset
|
|
case *fs.RangeOption:
|
|
offset, limit = x.Decode(int64(len(data)))
|
|
default:
|
|
if option.Mandatory() {
|
|
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
|
}
|
|
}
|
|
}
|
|
if l := int64(len(data)); offset > l {
|
|
offset = l
|
|
}
|
|
data = data[offset:]
|
|
if limit != -1 && limit < int64(len(data)) {
|
|
data = data[:limit]
|
|
}
|
|
|
|
return ioutil.NopCloser(bytes.NewReader(data)), nil
|
|
}
|
|
|
|
func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
|
|
src fs.ObjectInfo) (info *drive.File, err error) {
|
|
// Make the API request to upload metadata and file data.
|
|
size := src.Size()
|
|
if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
|
|
// Don't retry, return a retry error instead
|
|
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
|
info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo).
|
|
Media(in, googleapi.ContentType(uploadMimeType), googleapi.ChunkSize(0)).
|
|
Fields(partialFields).
|
|
SupportsAllDrives(true).
|
|
KeepRevisionForever(o.fs.opt.KeepRevisionForever).
|
|
Context(ctx).Do()
|
|
return o.fs.shouldRetry(ctx, err)
|
|
})
|
|
return
|
|
}
|
|
// Upload the file in chunks
|
|
return o.fs.Upload(ctx, in, size, uploadMimeType, o.id, o.remote, updateInfo)
|
|
}
|
|
|
|
// Update the already existing object
|
|
//
|
|
// Copy the reader into the object updating modTime and size.
|
|
//
|
|
// The new object may have been created if an error is returned
|
|
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
// If o is a shortcut
|
|
if isShortcutID(o.id) {
|
|
// Delete it first
|
|
err := o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Then put the file as a new file
|
|
newObj, err := o.fs.PutUnchecked(ctx, in, src, options...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Update the object
|
|
if newO, ok := newObj.(*Object); ok {
|
|
*o = *newO
|
|
} else {
|
|
fs.Debugf(newObj, "Failed to update object %T from new object %T", o, newObj)
|
|
}
|
|
return nil
|
|
}
|
|
srcMimeType := fs.MimeType(ctx, src)
|
|
updateInfo := &drive.File{
|
|
MimeType: srcMimeType,
|
|
ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
|
|
}
|
|
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch newO := newO.(type) {
|
|
case *Object:
|
|
*o = *newO
|
|
default:
|
|
return errors.New("object type changed by update")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
srcMimeType := fs.MimeType(ctx, src)
|
|
importMimeType := ""
|
|
updateInfo := &drive.File{
|
|
MimeType: srcMimeType,
|
|
ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
|
|
}
|
|
|
|
if o.fs.importMimeTypes == nil || o.fs.opt.SkipGdocs {
|
|
return fmt.Errorf("can't update google document type without --drive-import-formats")
|
|
}
|
|
importMimeType = o.fs.findImportFormat(ctx, updateInfo.MimeType)
|
|
if importMimeType == "" {
|
|
return fmt.Errorf("no import format found for %q", srcMimeType)
|
|
}
|
|
if importMimeType != o.documentMimeType {
|
|
return fmt.Errorf("can't change google document type (o: %q, src: %q, import: %q)", o.documentMimeType, srcMimeType, importMimeType)
|
|
}
|
|
updateInfo.MimeType = importMimeType
|
|
|
|
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
remote := src.Remote()
|
|
remote = remote[:len(remote)-o.extLen]
|
|
|
|
newO, err := o.fs.newObjectWithInfo(ctx, remote, info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch newO := newO.(type) {
|
|
case *documentObject:
|
|
*o = *newO
|
|
default:
|
|
return errors.New("object type changed by update")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (o *linkObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
|
return errors.New("cannot update link files")
|
|
}
|
|
|
|
// Remove an object
|
|
func (o *baseObject) Remove(ctx context.Context) error {
|
|
if len(o.parents) > 1 {
|
|
return errors.New("can't delete safely - has multiple parents")
|
|
}
|
|
return o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
|
|
}
|
|
|
|
// MimeType of an Object if known, "" otherwise
|
|
func (o *baseObject) MimeType(ctx context.Context) string {
|
|
return o.mimeType
|
|
}
|
|
|
|
// ID returns the ID of the Object if known, or "" if not
|
|
func (o *baseObject) ID() string {
|
|
return o.id
|
|
}
|
|
|
|
// ParentID returns the ID of the Object parent if known, or "" if not
|
|
func (o *baseObject) ParentID() string {
|
|
if len(o.parents) > 0 {
|
|
return o.parents[0]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (o *documentObject) ext() string {
|
|
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
|
|
}
|
|
func (o *linkObject) ext() string {
|
|
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
|
|
}
|
|
|
|
// templates for document link files
|
|
const (
|
|
urlTemplate = `[InternetShortcut]{{"\r"}}
|
|
URL={{ .URL }}{{"\r"}}
|
|
`
|
|
weblocTemplate = `<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0">
|
|
<dict>
|
|
<key>URL</key>
|
|
<string>{{ .URL }}</string>
|
|
</dict>
|
|
</plist>
|
|
`
|
|
desktopTemplate = `[Desktop Entry]
|
|
Encoding=UTF-8
|
|
Name={{ .Title }}
|
|
URL={{ .URL }}
|
|
Icon={{ .XDGIcon }}
|
|
Type=Link
|
|
`
|
|
htmlTemplate = `<html>
|
|
<head>
|
|
<meta http-equiv="refresh" content="0; url={{ .URL }}" />
|
|
<title>{{ .Title }}</title>
|
|
</head>
|
|
<body>
|
|
Loading <a href="{{ .URL }}">{{ .Title }}</a>
|
|
</body>
|
|
</html>
|
|
`
|
|
)
|
|
|
|
// Check the interfaces are satisfied
|
|
var (
|
|
_ fs.Fs = (*Fs)(nil)
|
|
_ fs.Purger = (*Fs)(nil)
|
|
_ fs.CleanUpper = (*Fs)(nil)
|
|
_ fs.PutStreamer = (*Fs)(nil)
|
|
_ fs.Copier = (*Fs)(nil)
|
|
_ fs.Mover = (*Fs)(nil)
|
|
_ fs.DirMover = (*Fs)(nil)
|
|
_ fs.Commander = (*Fs)(nil)
|
|
_ fs.DirCacheFlusher = (*Fs)(nil)
|
|
_ fs.ChangeNotifier = (*Fs)(nil)
|
|
_ fs.PutUncheckeder = (*Fs)(nil)
|
|
_ fs.PublicLinker = (*Fs)(nil)
|
|
_ fs.ListRer = (*Fs)(nil)
|
|
_ fs.MergeDirser = (*Fs)(nil)
|
|
_ fs.Abouter = (*Fs)(nil)
|
|
_ fs.Object = (*Object)(nil)
|
|
_ fs.MimeTyper = (*Object)(nil)
|
|
_ fs.IDer = (*Object)(nil)
|
|
_ fs.ParentIDer = (*Object)(nil)
|
|
_ fs.Object = (*documentObject)(nil)
|
|
_ fs.MimeTyper = (*documentObject)(nil)
|
|
_ fs.IDer = (*documentObject)(nil)
|
|
_ fs.ParentIDer = (*documentObject)(nil)
|
|
_ fs.Object = (*linkObject)(nil)
|
|
_ fs.MimeTyper = (*linkObject)(nil)
|
|
_ fs.IDer = (*linkObject)(nil)
|
|
_ fs.ParentIDer = (*linkObject)(nil)
|
|
)
|