2017-02-09 12:01:20 +01:00
|
|
|
package fs
|
|
|
|
|
|
|
|
import (
|
2020-11-05 12:33:32 +01:00
|
|
|
"context"
|
onedrive: add metadata support
This change adds support for metadata on OneDrive. Metadata (including
permissions) is supported for both files and directories.
OneDrive supports System Metadata (not User Metadata, as of this writing.) Much
of the metadata is read-only, and there are some differences between OneDrive
Personal and Business (see table in OneDrive backend docs for details).
Permissions are also supported, if --onedrive-metadata-permissions is set. The
accepted values for --onedrive-metadata-permissions are read, write, read,write, and
off (the default). write supports adding new permissions, updating the "role" of
existing permissions, and removing permissions. Updating and removing require
the Permission ID to be known, so it is recommended to use read,write instead of
write if you wish to update/remove permissions.
Permissions are read/written in JSON format using the same schema as the
OneDrive API, which differs slightly between OneDrive Personal and Business.
(See OneDrive backend docs for examples.)
To write permissions, pass in a "permissions" metadata key using this same
format. The --metadata-mapper tool can be very helpful for this.
When adding permissions, an email address can be provided in the User.ID or
DisplayName properties of grantedTo or grantedToIdentities. Alternatively, an
ObjectID can be provided in User.ID. At least one valid recipient must be
provided in order to add a permission for a user. Creating a Public Link is also
supported, if Link.Scope is set to "anonymous".
Note that adding a permission can fail if a conflicting permission already
exists for the file/folder.
To update an existing permission, include both the Permission ID and the new
roles to be assigned. roles is the only property that can be changed.
To remove permissions, pass in a blob containing only the permissions you wish
to keep (which can be empty, to remove all.)
Note that both reading and writing permissions requires extra API calls, so if
you don't need to read or write permissions it is recommended to omit --onedrive-
metadata-permissions.
Metadata and permissions are supported for Folders (directories) as well as
Files. Note that setting the mtime or btime on a Folder requires one extra API
call on OneDrive Business only.
OneDrive does not currently support User Metadata. When writing metadata, only
writeable system properties will be written -- any read-only or unrecognized keys
passed in will be ignored.
TIP: to see the metadata and permissions for any file or folder, run:
rclone lsjson remote:path --stat -M --onedrive-metadata-permissions read
See the OneDrive backend docs for a table of all the supported metadata
properties.
2024-02-22 15:17:14 +01:00
|
|
|
"encoding/json"
|
2017-02-09 12:01:20 +01:00
|
|
|
"fmt"
|
|
|
|
"log"
|
2021-08-18 13:48:44 +02:00
|
|
|
"os"
|
2017-06-26 23:46:45 +02:00
|
|
|
|
2019-07-26 02:54:09 +02:00
|
|
|
"github.com/sirupsen/logrus"
|
2017-02-09 12:01:20 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// LogLevel describes rclone's logs. These are a subset of the syslog log levels.
|
2023-09-27 16:31:47 +02:00
|
|
|
type LogLevel = Enum[logLevelChoices]
|
2017-02-09 12:01:20 +01:00
|
|
|
|
2017-02-10 14:28:06 +01:00
|
|
|
// Log levels. These are the syslog levels of which we only use a
|
|
|
|
// subset.
|
|
|
|
//
|
2022-08-05 17:35:41 +02:00
|
|
|
// LOG_EMERG system is unusable
|
|
|
|
// LOG_ALERT action must be taken immediately
|
|
|
|
// LOG_CRIT critical conditions
|
|
|
|
// LOG_ERR error conditions
|
|
|
|
// LOG_WARNING warning conditions
|
|
|
|
// LOG_NOTICE normal, but significant, condition
|
|
|
|
// LOG_INFO informational message
|
|
|
|
// LOG_DEBUG debug-level message
|
2017-02-09 12:01:20 +01:00
|
|
|
const (
|
2017-02-09 22:22:46 +01:00
|
|
|
LogLevelEmergency LogLevel = iota
|
2017-02-09 12:01:20 +01:00
|
|
|
LogLevelAlert
|
|
|
|
LogLevelCritical
|
|
|
|
LogLevelError // Error - can't be suppressed
|
|
|
|
LogLevelWarning
|
|
|
|
LogLevelNotice // Normal logging, -q suppresses
|
|
|
|
LogLevelInfo // Transfers, needs -v
|
|
|
|
LogLevelDebug // Debug level, needs -vv
|
|
|
|
)
|
|
|
|
|
2023-09-27 16:31:47 +02:00
|
|
|
type logLevelChoices struct{}
|
|
|
|
|
|
|
|
func (logLevelChoices) Choices() []string {
|
|
|
|
return []string{
|
|
|
|
LogLevelEmergency: "EMERGENCY",
|
|
|
|
LogLevelAlert: "ALERT",
|
|
|
|
LogLevelCritical: "CRITICAL",
|
|
|
|
LogLevelError: "ERROR",
|
|
|
|
LogLevelWarning: "WARNING",
|
|
|
|
LogLevelNotice: "NOTICE",
|
|
|
|
LogLevelInfo: "INFO",
|
|
|
|
LogLevelDebug: "DEBUG",
|
2017-02-10 14:28:06 +01:00
|
|
|
}
|
|
|
|
}
|
2017-02-09 12:01:20 +01:00
|
|
|
|
2023-09-27 16:31:47 +02:00
|
|
|
func (logLevelChoices) Type() string {
|
2023-09-27 14:58:27 +02:00
|
|
|
return "LogLevel"
|
2017-06-26 23:46:45 +02:00
|
|
|
}
|
|
|
|
|
2021-08-18 13:48:44 +02:00
|
|
|
// LogPrintPid enables process pid in log
|
|
|
|
var LogPrintPid = false
|
|
|
|
|
2024-07-09 19:42:01 +02:00
|
|
|
// InstallJSONLogger is a hook that --use-json-log calls
|
|
|
|
var InstallJSONLogger = func(logLevel LogLevel) {}
|
|
|
|
|
2024-06-07 12:42:52 +02:00
|
|
|
// LogOutput sends the text to the logger of level
|
|
|
|
var LogOutput = func(level LogLevel, text string) {
|
2020-12-17 12:55:27 +01:00
|
|
|
text = fmt.Sprintf("%-6s: %s", level, text)
|
2021-08-18 13:48:44 +02:00
|
|
|
if LogPrintPid {
|
|
|
|
text = fmt.Sprintf("[%d] %s", os.Getpid(), text)
|
|
|
|
}
|
2018-09-02 19:11:09 +02:00
|
|
|
_ = log.Output(4, text)
|
2017-02-10 14:28:06 +01:00
|
|
|
}
|
|
|
|
|
2020-04-11 19:02:50 +02:00
|
|
|
// LogValueItem describes keyed item for a JSON log entry
|
|
|
|
type LogValueItem struct {
|
2020-12-30 14:06:43 +01:00
|
|
|
key string
|
|
|
|
value interface{}
|
|
|
|
render bool
|
2020-04-11 19:02:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// LogValue should be used as an argument to any logging calls to
|
|
|
|
// augment the JSON output with more structured information.
|
|
|
|
//
|
|
|
|
// key is the dictionary parameter used to store value.
|
|
|
|
func LogValue(key string, value interface{}) LogValueItem {
|
2020-12-30 14:06:43 +01:00
|
|
|
return LogValueItem{key: key, value: value, render: true}
|
2020-04-11 19:02:50 +02:00
|
|
|
}
|
|
|
|
|
2020-12-30 14:06:43 +01:00
|
|
|
// LogValueHide should be used as an argument to any logging calls to
|
|
|
|
// augment the JSON output with more structured information.
|
|
|
|
//
|
|
|
|
// key is the dictionary parameter used to store value.
|
|
|
|
//
|
|
|
|
// String() will return a blank string - this is useful to put items
|
|
|
|
// in which don't print into the log.
|
|
|
|
func LogValueHide(key string, value interface{}) LogValueItem {
|
|
|
|
return LogValueItem{key: key, value: value, render: false}
|
|
|
|
}
|
|
|
|
|
|
|
|
// String returns the representation of value. If render is fals this
|
|
|
|
// is an empty string so LogValueItem entries won't show in the
|
|
|
|
// textual representation of logs.
|
2020-04-11 19:02:50 +02:00
|
|
|
func (j LogValueItem) String() string {
|
2020-12-30 14:06:43 +01:00
|
|
|
if !j.render {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
if do, ok := j.value.(fmt.Stringer); ok {
|
|
|
|
return do.String()
|
|
|
|
}
|
|
|
|
return fmt.Sprint(j.value)
|
2020-04-11 19:02:50 +02:00
|
|
|
}
|
|
|
|
|
2018-01-12 17:30:54 +01:00
|
|
|
// LogPrintf produces a log string from the arguments passed in
|
|
|
|
func LogPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
|
2017-02-09 12:01:20 +01:00
|
|
|
out := fmt.Sprintf(text, args...)
|
2019-08-02 16:57:09 +02:00
|
|
|
|
2020-11-05 12:33:32 +01:00
|
|
|
if GetConfig(context.TODO()).UseJSONLog {
|
2019-08-02 16:57:09 +02:00
|
|
|
fields := logrus.Fields{}
|
|
|
|
if o != nil {
|
|
|
|
fields = logrus.Fields{
|
|
|
|
"object": fmt.Sprintf("%+v", o),
|
|
|
|
"objectType": fmt.Sprintf("%T", o),
|
|
|
|
}
|
|
|
|
}
|
2020-04-11 19:02:50 +02:00
|
|
|
for _, arg := range args {
|
|
|
|
if item, ok := arg.(LogValueItem); ok {
|
|
|
|
fields[item.key] = item.value
|
|
|
|
}
|
2020-02-15 22:21:01 +01:00
|
|
|
}
|
2019-07-26 02:54:09 +02:00
|
|
|
switch level {
|
|
|
|
case LogLevelDebug:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Debug(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
case LogLevelInfo:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Info(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
case LogLevelNotice, LogLevelWarning:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Warn(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
case LogLevelError:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Error(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
case LogLevelCritical:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Fatal(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
case LogLevelEmergency, LogLevelAlert:
|
2019-08-02 16:57:09 +02:00
|
|
|
logrus.WithFields(fields).Panic(out)
|
2019-07-26 02:54:09 +02:00
|
|
|
}
|
|
|
|
} else {
|
2019-08-02 16:57:09 +02:00
|
|
|
if o != nil {
|
|
|
|
out = fmt.Sprintf("%v: %s", o, out)
|
|
|
|
}
|
2024-06-07 12:42:52 +02:00
|
|
|
LogOutput(level, out)
|
2019-07-26 02:54:09 +02:00
|
|
|
}
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
|
|
|
|
2017-06-26 23:46:45 +02:00
|
|
|
// LogLevelPrintf writes logs at the given level
|
|
|
|
func LogLevelPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
|
2020-11-05 12:33:32 +01:00
|
|
|
if GetConfig(context.TODO()).LogLevel >= level {
|
2018-01-12 17:30:54 +01:00
|
|
|
LogPrintf(level, o, text, args...)
|
2017-06-26 23:46:45 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-07 13:32:01 +02:00
|
|
|
// Panicf writes alert log output for this Object or Fs and calls panic().
|
|
|
|
// It should always be seen by the user.
|
|
|
|
func Panicf(o interface{}, text string, args ...interface{}) {
|
|
|
|
if GetConfig(context.TODO()).LogLevel >= LogLevelAlert {
|
|
|
|
LogPrintf(LogLevelAlert, o, text, args...)
|
|
|
|
}
|
|
|
|
panic(fmt.Sprintf(text, args...))
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fatalf writes critical log output for this Object or Fs and calls os.Exit(1).
|
|
|
|
// It should always be seen by the user.
|
|
|
|
func Fatalf(o interface{}, text string, args ...interface{}) {
|
|
|
|
if GetConfig(context.TODO()).LogLevel >= LogLevelCritical {
|
|
|
|
LogPrintf(LogLevelCritical, o, text, args...)
|
|
|
|
}
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
|
2017-02-09 12:01:20 +01:00
|
|
|
// Errorf writes error log output for this Object or Fs. It
|
2017-02-10 14:28:06 +01:00
|
|
|
// should always be seen by the user.
|
2017-02-09 12:01:20 +01:00
|
|
|
func Errorf(o interface{}, text string, args ...interface{}) {
|
2024-06-07 12:37:20 +02:00
|
|
|
LogLevelPrintf(LogLevelError, o, text, args...)
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
|
|
|
|
2024-06-07 13:32:01 +02:00
|
|
|
// Printf writes log output for this Object or Fs, same as Logf.
|
|
|
|
func Printf(o interface{}, text string, args ...interface{}) {
|
|
|
|
LogLevelPrintf(LogLevelNotice, o, text, args...)
|
|
|
|
}
|
|
|
|
|
2017-02-09 12:01:20 +01:00
|
|
|
// Logf writes log output for this Object or Fs. This should be
|
2021-06-21 17:30:26 +02:00
|
|
|
// considered to be Notice level logging. It is the default level.
|
|
|
|
// By default rclone should not log very much so only use this for
|
2017-02-09 22:22:46 +01:00
|
|
|
// important things the user should see. The user can filter these
|
|
|
|
// out with the -q flag.
|
2017-02-09 12:01:20 +01:00
|
|
|
func Logf(o interface{}, text string, args ...interface{}) {
|
2024-06-07 12:37:20 +02:00
|
|
|
LogLevelPrintf(LogLevelNotice, o, text, args...)
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
|
|
|
|
2017-02-09 22:22:46 +01:00
|
|
|
// Infof writes info on transfers for this Object or Fs. Use this
|
|
|
|
// level for logging transfers, deletions and things which should
|
|
|
|
// appear with the -v flag.
|
2017-02-09 12:01:20 +01:00
|
|
|
func Infof(o interface{}, text string, args ...interface{}) {
|
2024-06-07 12:37:20 +02:00
|
|
|
LogLevelPrintf(LogLevelInfo, o, text, args...)
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
|
|
|
|
2017-02-09 22:22:46 +01:00
|
|
|
// Debugf writes debugging output for this Object or Fs. Use this for
|
|
|
|
// debug only. The user must have to specify -vv to see this.
|
2017-02-09 12:01:20 +01:00
|
|
|
func Debugf(o interface{}, text string, args ...interface{}) {
|
2024-06-07 12:37:20 +02:00
|
|
|
LogLevelPrintf(LogLevelDebug, o, text, args...)
|
2017-05-09 12:38:58 +02:00
|
|
|
}
|
|
|
|
|
2018-01-12 17:30:54 +01:00
|
|
|
// LogDirName returns an object for the logger, logging a root
|
|
|
|
// directory which would normally be "" as the Fs
|
|
|
|
func LogDirName(f Fs, dir string) interface{} {
|
|
|
|
if dir != "" {
|
|
|
|
return dir
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
2018-01-12 17:30:54 +01:00
|
|
|
return f
|
2017-02-09 12:01:20 +01:00
|
|
|
}
|
onedrive: add metadata support
This change adds support for metadata on OneDrive. Metadata (including
permissions) is supported for both files and directories.
OneDrive supports System Metadata (not User Metadata, as of this writing.) Much
of the metadata is read-only, and there are some differences between OneDrive
Personal and Business (see table in OneDrive backend docs for details).
Permissions are also supported, if --onedrive-metadata-permissions is set. The
accepted values for --onedrive-metadata-permissions are read, write, read,write, and
off (the default). write supports adding new permissions, updating the "role" of
existing permissions, and removing permissions. Updating and removing require
the Permission ID to be known, so it is recommended to use read,write instead of
write if you wish to update/remove permissions.
Permissions are read/written in JSON format using the same schema as the
OneDrive API, which differs slightly between OneDrive Personal and Business.
(See OneDrive backend docs for examples.)
To write permissions, pass in a "permissions" metadata key using this same
format. The --metadata-mapper tool can be very helpful for this.
When adding permissions, an email address can be provided in the User.ID or
DisplayName properties of grantedTo or grantedToIdentities. Alternatively, an
ObjectID can be provided in User.ID. At least one valid recipient must be
provided in order to add a permission for a user. Creating a Public Link is also
supported, if Link.Scope is set to "anonymous".
Note that adding a permission can fail if a conflicting permission already
exists for the file/folder.
To update an existing permission, include both the Permission ID and the new
roles to be assigned. roles is the only property that can be changed.
To remove permissions, pass in a blob containing only the permissions you wish
to keep (which can be empty, to remove all.)
Note that both reading and writing permissions requires extra API calls, so if
you don't need to read or write permissions it is recommended to omit --onedrive-
metadata-permissions.
Metadata and permissions are supported for Folders (directories) as well as
Files. Note that setting the mtime or btime on a Folder requires one extra API
call on OneDrive Business only.
OneDrive does not currently support User Metadata. When writing metadata, only
writeable system properties will be written -- any read-only or unrecognized keys
passed in will be ignored.
TIP: to see the metadata and permissions for any file or folder, run:
rclone lsjson remote:path --stat -M --onedrive-metadata-permissions read
See the OneDrive backend docs for a table of all the supported metadata
properties.
2024-02-22 15:17:14 +01:00
|
|
|
|
|
|
|
// PrettyPrint formats JSON for improved readability in debug logs.
|
|
|
|
// If it can't Marshal JSON, it falls back to fmt.
|
|
|
|
func PrettyPrint(in any, label string, level LogLevel) {
|
|
|
|
if GetConfig(context.TODO()).LogLevel < level {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
inBytes, err := json.MarshalIndent(in, "", "\t")
|
2024-03-28 18:00:43 +01:00
|
|
|
if err != nil || string(inBytes) == "{}" || string(inBytes) == "[]" {
|
onedrive: add metadata support
This change adds support for metadata on OneDrive. Metadata (including
permissions) is supported for both files and directories.
OneDrive supports System Metadata (not User Metadata, as of this writing.) Much
of the metadata is read-only, and there are some differences between OneDrive
Personal and Business (see table in OneDrive backend docs for details).
Permissions are also supported, if --onedrive-metadata-permissions is set. The
accepted values for --onedrive-metadata-permissions are read, write, read,write, and
off (the default). write supports adding new permissions, updating the "role" of
existing permissions, and removing permissions. Updating and removing require
the Permission ID to be known, so it is recommended to use read,write instead of
write if you wish to update/remove permissions.
Permissions are read/written in JSON format using the same schema as the
OneDrive API, which differs slightly between OneDrive Personal and Business.
(See OneDrive backend docs for examples.)
To write permissions, pass in a "permissions" metadata key using this same
format. The --metadata-mapper tool can be very helpful for this.
When adding permissions, an email address can be provided in the User.ID or
DisplayName properties of grantedTo or grantedToIdentities. Alternatively, an
ObjectID can be provided in User.ID. At least one valid recipient must be
provided in order to add a permission for a user. Creating a Public Link is also
supported, if Link.Scope is set to "anonymous".
Note that adding a permission can fail if a conflicting permission already
exists for the file/folder.
To update an existing permission, include both the Permission ID and the new
roles to be assigned. roles is the only property that can be changed.
To remove permissions, pass in a blob containing only the permissions you wish
to keep (which can be empty, to remove all.)
Note that both reading and writing permissions requires extra API calls, so if
you don't need to read or write permissions it is recommended to omit --onedrive-
metadata-permissions.
Metadata and permissions are supported for Folders (directories) as well as
Files. Note that setting the mtime or btime on a Folder requires one extra API
call on OneDrive Business only.
OneDrive does not currently support User Metadata. When writing metadata, only
writeable system properties will be written -- any read-only or unrecognized keys
passed in will be ignored.
TIP: to see the metadata and permissions for any file or folder, run:
rclone lsjson remote:path --stat -M --onedrive-metadata-permissions read
See the OneDrive backend docs for a table of all the supported metadata
properties.
2024-02-22 15:17:14 +01:00
|
|
|
LogPrintf(level, label, "\n%+v\n", in)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
LogPrintf(level, label, "\n%s\n", string(inBytes))
|
|
|
|
}
|