diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index b86328b20..c09848166 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -3,62 +3,26 @@ package lsjson import ( "encoding/json" "fmt" - "log" "os" - "path" - "time" - "github.com/ncw/rclone/backend/crypt" "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/ls/lshelp" - "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/operations" - "github.com/ncw/rclone/fs/walk" "github.com/pkg/errors" "github.com/spf13/cobra" ) var ( - recurse bool - showHash bool - showEncrypted bool - showOrigIDs bool - noModTime bool + opt operations.ListJSONOpt ) func init() { cmd.Root.AddCommand(commandDefintion) - commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.") - commandDefintion.Flags().BoolVarP(&showHash, "hash", "", false, "Include hashes in the output (may take longer).") - commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).") - commandDefintion.Flags().BoolVarP(&showEncrypted, "encrypted", "M", false, "Show the encrypted names.") - commandDefintion.Flags().BoolVarP(&showOrigIDs, "original", "", false, "Show the ID of the underlying Object.") -} - -// lsJSON in the struct which gets marshalled for each line -type lsJSON struct { - Path string - Name string - Encrypted string `json:",omitempty"` - Size int64 - MimeType string `json:",omitempty"` - ModTime Timestamp //`json:",omitempty"` - IsDir bool - Hashes map[string]string `json:",omitempty"` - ID string `json:",omitempty"` - OrigID string `json:",omitempty"` -} - -// Timestamp a time in RFC3339 format with Nanosecond precision secongs -type Timestamp time.Time - -// MarshalJSON turns a Timestamp into JSON -func (t Timestamp) MarshalJSON() (out []byte, err error) { - tt := time.Time(t) - if tt.IsZero() { - return []byte(`""`), nil - } - return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil + commandDefintion.Flags().BoolVarP(&opt.Recurse, "recursive", "R", false, "Recurse into the listing.") + commandDefintion.Flags().BoolVarP(&opt.ShowHash, "hash", "", false, "Include hashes in the output (may take longer).") + commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).") + commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.") + commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.") } var commandDefintion = &cobra.Command{ @@ -104,107 +68,27 @@ can be processed line by line as each item is written one to a line. Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) fsrc := cmd.NewFsSrc(args) - var cipher crypt.Cipher - if showEncrypted { - fsInfo, _, _, config, err := fs.ConfigFs(args[0]) - if err != nil { - log.Fatalf(err.Error()) - } - if fsInfo.Name != "crypt" { - log.Fatalf("The remote needs to be of type \"crypt\"") - } - cipher, err = crypt.NewCipher(config) - if err != nil { - log.Fatalf(err.Error()) - } - } cmd.Run(false, false, command, func() error { fmt.Println("[") first := true - err := walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(dirPath string, entries fs.DirEntries, err error) error { + err := operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error { + out, err := json.Marshal(item) if err != nil { - fs.CountError(err) - fs.Errorf(dirPath, "error listing: %v", err) - return nil + return errors.Wrap(err, "failed to marshal list object") } - for _, entry := range entries { - item := lsJSON{ - Path: entry.Remote(), - Name: path.Base(entry.Remote()), - Size: entry.Size(), - MimeType: fs.MimeTypeDirEntry(entry), - } - if !noModTime { - item.ModTime = Timestamp(entry.ModTime()) - } - if cipher != nil { - switch entry.(type) { - case fs.Directory: - item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote())) - case fs.Object: - item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote())) - default: - fs.Errorf(nil, "Unknown type %T in listing", entry) - } - } - if do, ok := entry.(fs.IDer); ok { - item.ID = do.ID() - } - if showOrigIDs { - cur := entry - for { - u, ok := cur.(fs.ObjectUnWrapper) - if !ok { - break // not a wrapped object, use current id - } - next := u.UnWrap() - if next == nil { - break // no base object found, use current id - } - cur = next - } - if do, ok := cur.(fs.IDer); ok { - item.OrigID = do.ID() - } - } - switch x := entry.(type) { - case fs.Directory: - item.IsDir = true - case fs.Object: - item.IsDir = false - if showHash { - item.Hashes = make(map[string]string) - for _, hashType := range x.Fs().Hashes().Array() { - hash, err := x.Hash(hashType) - if err != nil { - fs.Errorf(x, "Failed to read hash: %v", err) - } else if hash != "" { - item.Hashes[hashType.String()] = hash - } - } - } - default: - fs.Errorf(nil, "Unknown type %T in listing", entry) - } - out, err := json.Marshal(item) - if err != nil { - return errors.Wrap(err, "failed to marshal list object") - } - if first { - first = false - } else { - fmt.Print(",\n") - } - _, err = os.Stdout.Write(out) - if err != nil { - return errors.Wrap(err, "failed to write to output") - } - + if first { + first = false + } else { + fmt.Print(",\n") + } + _, err = os.Stdout.Write(out) + if err != nil { + return errors.Wrap(err, "failed to write to output") } return nil }) if err != nil { - return errors.Wrap(err, "error listing JSON") + return err } if !first { fmt.Println() diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go new file mode 100644 index 000000000..79465eceb --- /dev/null +++ b/fs/operations/lsjson.go @@ -0,0 +1,141 @@ +package operations + +import ( + "path" + "time" + + "github.com/ncw/rclone/backend/crypt" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/walk" + "github.com/pkg/errors" +) + +// ListJSONItem in the struct which gets marshalled for each line +type ListJSONItem struct { + Path string + Name string + Encrypted string `json:",omitempty"` + Size int64 + MimeType string `json:",omitempty"` + ModTime Timestamp //`json:",omitempty"` + IsDir bool + Hashes map[string]string `json:",omitempty"` + ID string `json:",omitempty"` + OrigID string `json:",omitempty"` +} + +// Timestamp a time in RFC3339 format with Nanosecond precision secongs +type Timestamp time.Time + +// MarshalJSON turns a Timestamp into JSON +func (t Timestamp) MarshalJSON() (out []byte, err error) { + tt := time.Time(t) + if tt.IsZero() { + return []byte(`""`), nil + } + return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil +} + +// ListJSONOpt describes the options for ListJSON +type ListJSONOpt struct { + Recurse bool `json:"recurse"` + NoModTime bool `json:"noModTime"` + ShowEncrypted bool `json:"showEncrypted"` + ShowOrigIDs bool `json:"showOrigIDs"` + ShowHash bool `json:"showHash"` +} + +// ListJSON lists fsrc using the options in opt calling callback for each item +func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error { + var cipher crypt.Cipher + if opt.ShowEncrypted { + fsInfo, _, _, config, err := fs.ConfigFs(fsrc.Name() + ":" + fsrc.Root()) + if err != nil { + return errors.Wrap(err, "ListJSON failed to load config for crypt remote") + } + if fsInfo.Name != "crypt" { + return errors.New("The remote needs to be of type \"crypt\"") + } + cipher, err = crypt.NewCipher(config) + if err != nil { + return errors.Wrap(err, "ListJSON failed to make new crypt remote") + } + } + err := walk.Walk(fsrc, remote, false, ConfigMaxDepth(opt.Recurse), func(dirPath string, entries fs.DirEntries, err error) error { + if err != nil { + fs.CountError(err) + fs.Errorf(dirPath, "error listing: %v", err) + return nil + } + for _, entry := range entries { + item := ListJSONItem{ + Path: entry.Remote(), + Name: path.Base(entry.Remote()), + Size: entry.Size(), + MimeType: fs.MimeTypeDirEntry(entry), + } + if !opt.NoModTime { + item.ModTime = Timestamp(entry.ModTime()) + } + if cipher != nil { + switch entry.(type) { + case fs.Directory: + item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote())) + case fs.Object: + item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote())) + default: + fs.Errorf(nil, "Unknown type %T in listing", entry) + } + } + if do, ok := entry.(fs.IDer); ok { + item.ID = do.ID() + } + if opt.ShowOrigIDs { + cur := entry + for { + u, ok := cur.(fs.ObjectUnWrapper) + if !ok { + break // not a wrapped object, use current id + } + next := u.UnWrap() + if next == nil { + break // no base object found, use current id + } + cur = next + } + if do, ok := cur.(fs.IDer); ok { + item.OrigID = do.ID() + } + } + switch x := entry.(type) { + case fs.Directory: + item.IsDir = true + case fs.Object: + item.IsDir = false + if opt.ShowHash { + item.Hashes = make(map[string]string) + for _, hashType := range x.Fs().Hashes().Array() { + hash, err := x.Hash(hashType) + if err != nil { + fs.Errorf(x, "Failed to read hash: %v", err) + } else if hash != "" { + item.Hashes[hashType.String()] = hash + } + } + } + default: + fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry) + } + err = callback(&item) + if err != nil { + return errors.Wrap(err, "callback failed in ListJSON") + } + + } + return nil + }) + if err != nil { + return errors.Wrap(err, "error in ListJSON") + } + return nil +}