mirror of
https://github.com/rclone/rclone
synced 2024-11-24 01:26:25 +01:00
lsjson: factor internals of lsjson command into operations
This commit is contained in:
Nick Craig-Wood
parent
fa0a9653d2
commit
fc2afcbcbd
@ -3,62 +3,26 @@ package lsjson
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/backend/crypt"
|
||||
"github.com/ncw/rclone/cmd"
|
||||
"github.com/ncw/rclone/cmd/ls/lshelp"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/operations"
|
||||
"github.com/ncw/rclone/fs/walk"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
recurse bool
|
||||
showHash bool
|
||||
showEncrypted bool
|
||||
showOrigIDs bool
|
||||
noModTime bool
|
||||
opt operations.ListJSONOpt
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefintion)
|
||||
commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.")
|
||||
commandDefintion.Flags().BoolVarP(&showHash, "hash", "", false, "Include hashes in the output (may take longer).")
|
||||
commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
|
||||
commandDefintion.Flags().BoolVarP(&showEncrypted, "encrypted", "M", false, "Show the encrypted names.")
|
||||
commandDefintion.Flags().BoolVarP(&showOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
|
||||
}
|
||||
|
||||
// lsJSON in the struct which gets marshalled for each line
|
||||
type lsJSON struct {
|
||||
Path string
|
||||
Name string
|
||||
Encrypted string `json:",omitempty"`
|
||||
Size int64
|
||||
MimeType string `json:",omitempty"`
|
||||
ModTime Timestamp //`json:",omitempty"`
|
||||
IsDir bool
|
||||
Hashes map[string]string `json:",omitempty"`
|
||||
ID string `json:",omitempty"`
|
||||
OrigID string `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Timestamp a time in RFC3339 format with Nanosecond precision secongs
|
||||
type Timestamp time.Time
|
||||
|
||||
// MarshalJSON turns a Timestamp into JSON
|
||||
func (t Timestamp) MarshalJSON() (out []byte, err error) {
|
||||
tt := time.Time(t)
|
||||
if tt.IsZero() {
|
||||
return []byte(`""`), nil
|
||||
}
|
||||
return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil
|
||||
commandDefintion.Flags().BoolVarP(&opt.Recurse, "recursive", "R", false, "Recurse into the listing.")
|
||||
commandDefintion.Flags().BoolVarP(&opt.ShowHash, "hash", "", false, "Include hashes in the output (may take longer).")
|
||||
commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
|
||||
commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.")
|
||||
commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
|
||||
}
|
||||
|
||||
var commandDefintion = &cobra.Command{
|
||||
@ -104,107 +68,27 @@ can be processed line by line as each item is written one to a line.
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
var cipher crypt.Cipher
|
||||
if showEncrypted {
|
||||
fsInfo, _, _, config, err := fs.ConfigFs(args[0])
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
if fsInfo.Name != "crypt" {
|
||||
log.Fatalf("The remote needs to be of type \"crypt\"")
|
||||
}
|
||||
cipher, err = crypt.NewCipher(config)
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
}
|
||||
cmd.Run(false, false, command, func() error {
|
||||
fmt.Println("[")
|
||||
first := true
|
||||
err := walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
err := operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
|
||||
out, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
fs.CountError(err)
|
||||
fs.Errorf(dirPath, "error listing: %v", err)
|
||||
return nil
|
||||
return errors.Wrap(err, "failed to marshal list object")
|
||||
}
|
||||
for _, entry := range entries {
|
||||
item := lsJSON{
|
||||
Path: entry.Remote(),
|
||||
Name: path.Base(entry.Remote()),
|
||||
Size: entry.Size(),
|
||||
MimeType: fs.MimeTypeDirEntry(entry),
|
||||
}
|
||||
if !noModTime {
|
||||
item.ModTime = Timestamp(entry.ModTime())
|
||||
}
|
||||
if cipher != nil {
|
||||
switch entry.(type) {
|
||||
case fs.Directory:
|
||||
item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote()))
|
||||
case fs.Object:
|
||||
item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote()))
|
||||
default:
|
||||
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
||||
}
|
||||
}
|
||||
if do, ok := entry.(fs.IDer); ok {
|
||||
item.ID = do.ID()
|
||||
}
|
||||
if showOrigIDs {
|
||||
cur := entry
|
||||
for {
|
||||
u, ok := cur.(fs.ObjectUnWrapper)
|
||||
if !ok {
|
||||
break // not a wrapped object, use current id
|
||||
}
|
||||
next := u.UnWrap()
|
||||
if next == nil {
|
||||
break // no base object found, use current id
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
if do, ok := cur.(fs.IDer); ok {
|
||||
item.OrigID = do.ID()
|
||||
}
|
||||
}
|
||||
switch x := entry.(type) {
|
||||
case fs.Directory:
|
||||
item.IsDir = true
|
||||
case fs.Object:
|
||||
item.IsDir = false
|
||||
if showHash {
|
||||
item.Hashes = make(map[string]string)
|
||||
for _, hashType := range x.Fs().Hashes().Array() {
|
||||
hash, err := x.Hash(hashType)
|
||||
if err != nil {
|
||||
fs.Errorf(x, "Failed to read hash: %v", err)
|
||||
} else if hash != "" {
|
||||
item.Hashes[hashType.String()] = hash
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
||||
}
|
||||
out, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to marshal list object")
|
||||
}
|
||||
if first {
|
||||
first = false
|
||||
} else {
|
||||
fmt.Print(",\n")
|
||||
}
|
||||
_, err = os.Stdout.Write(out)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write to output")
|
||||
}
|
||||
|
||||
if first {
|
||||
first = false
|
||||
} else {
|
||||
fmt.Print(",\n")
|
||||
}
|
||||
_, err = os.Stdout.Write(out)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write to output")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error listing JSON")
|
||||
return err
|
||||
}
|
||||
if !first {
|
||||
fmt.Println()
|
||||
|
||||
141
fs/operations/lsjson.go
Normal file
141
fs/operations/lsjson.go
Normal file
@ -0,0 +1,141 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/backend/crypt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/walk"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ListJSONItem in the struct which gets marshalled for each line
|
||||
type ListJSONItem struct {
|
||||
Path string
|
||||
Name string
|
||||
Encrypted string `json:",omitempty"`
|
||||
Size int64
|
||||
MimeType string `json:",omitempty"`
|
||||
ModTime Timestamp //`json:",omitempty"`
|
||||
IsDir bool
|
||||
Hashes map[string]string `json:",omitempty"`
|
||||
ID string `json:",omitempty"`
|
||||
OrigID string `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Timestamp a time in RFC3339 format with Nanosecond precision secongs
|
||||
type Timestamp time.Time
|
||||
|
||||
// MarshalJSON turns a Timestamp into JSON
|
||||
func (t Timestamp) MarshalJSON() (out []byte, err error) {
|
||||
tt := time.Time(t)
|
||||
if tt.IsZero() {
|
||||
return []byte(`""`), nil
|
||||
}
|
||||
return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil
|
||||
}
|
||||
|
||||
// ListJSONOpt describes the options for ListJSON
|
||||
type ListJSONOpt struct {
|
||||
Recurse bool `json:"recurse"`
|
||||
NoModTime bool `json:"noModTime"`
|
||||
ShowEncrypted bool `json:"showEncrypted"`
|
||||
ShowOrigIDs bool `json:"showOrigIDs"`
|
||||
ShowHash bool `json:"showHash"`
|
||||
}
|
||||
|
||||
// ListJSON lists fsrc using the options in opt calling callback for each item
|
||||
func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
|
||||
var cipher crypt.Cipher
|
||||
if opt.ShowEncrypted {
|
||||
fsInfo, _, _, config, err := fs.ConfigFs(fsrc.Name() + ":" + fsrc.Root())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "ListJSON failed to load config for crypt remote")
|
||||
}
|
||||
if fsInfo.Name != "crypt" {
|
||||
return errors.New("The remote needs to be of type \"crypt\"")
|
||||
}
|
||||
cipher, err = crypt.NewCipher(config)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "ListJSON failed to make new crypt remote")
|
||||
}
|
||||
}
|
||||
err := walk.Walk(fsrc, remote, false, ConfigMaxDepth(opt.Recurse), func(dirPath string, entries fs.DirEntries, err error) error {
|
||||
if err != nil {
|
||||
fs.CountError(err)
|
||||
fs.Errorf(dirPath, "error listing: %v", err)
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
item := ListJSONItem{
|
||||
Path: entry.Remote(),
|
||||
Name: path.Base(entry.Remote()),
|
||||
Size: entry.Size(),
|
||||
MimeType: fs.MimeTypeDirEntry(entry),
|
||||
}
|
||||
if !opt.NoModTime {
|
||||
item.ModTime = Timestamp(entry.ModTime())
|
||||
}
|
||||
if cipher != nil {
|
||||
switch entry.(type) {
|
||||
case fs.Directory:
|
||||
item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote()))
|
||||
case fs.Object:
|
||||
item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote()))
|
||||
default:
|
||||
fs.Errorf(nil, "Unknown type %T in listing", entry)
|
||||
}
|
||||
}
|
||||
if do, ok := entry.(fs.IDer); ok {
|
||||
item.ID = do.ID()
|
||||
}
|
||||
if opt.ShowOrigIDs {
|
||||
cur := entry
|
||||
for {
|
||||
u, ok := cur.(fs.ObjectUnWrapper)
|
||||
if !ok {
|
||||
break // not a wrapped object, use current id
|
||||
}
|
||||
next := u.UnWrap()
|
||||
if next == nil {
|
||||
break // no base object found, use current id
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
if do, ok := cur.(fs.IDer); ok {
|
||||
item.OrigID = do.ID()
|
||||
}
|
||||
}
|
||||
switch x := entry.(type) {
|
||||
case fs.Directory:
|
||||
item.IsDir = true
|
||||
case fs.Object:
|
||||
item.IsDir = false
|
||||
if opt.ShowHash {
|
||||
item.Hashes = make(map[string]string)
|
||||
for _, hashType := range x.Fs().Hashes().Array() {
|
||||
hash, err := x.Hash(hashType)
|
||||
if err != nil {
|
||||
fs.Errorf(x, "Failed to read hash: %v", err)
|
||||
} else if hash != "" {
|
||||
item.Hashes[hashType.String()] = hash
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry)
|
||||
}
|
||||
err = callback(&item)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "callback failed in ListJSON")
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error in ListJSON")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user