github/rclone
1
Fork 0
mirror of https://github.com/rclone/rclone synced 2024-12-23 14:23:44 +01:00
Code Releases Activity

hashsum: Add flag --base64 flag - fixes #3663

Browse Source 
This flag can be used to output QuickXorHash in the same format as MS
Graph API.
This commit is contained in:
landall 2019-10-27 03:27:33 +08:00 committed by Nick Craig-Wood
parent 18d26e2ddb
commit 77e55b8265
3 changed files with 67 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/hashsum/hashsum.go
View File

@ -7,13 +7,20 @@ import (
"os"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
var (
outputBase64 = false
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &outputBase64, "base64", "", outputBase64, "Output base64 encoded hashsum")
}
var commandDefinition = &cobra.Command{
@ -55,6 +62,9 @@ Then
}
fsrc := cmd.NewFsSrc(args[1:])
cmd.Run(false, false, command, func() error {
if outputBase64 {
return operations.HashListerBase64(context.Background(), ht, fsrc, os.Stdout)
}
return operations.HashLister(context.Background(), ht, fsrc, os.Stdout)
})
return nil

24
fs/operations/operations.go
View File

@ -4,7 +4,9 @@ package operations
import (
"bytes"
"context"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
@ -1042,8 +1044,9 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error {
}
// hashSum returns the human readable hash for ht passed in. This may
// be UNSUPPORTED or ERROR.
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will
// return an error.
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) (string, error) {
var err error
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
defer func() {
@ -1056,17 +1059,30 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
fs.Debugf(o, "Failed to read %v: %v", ht, err)
sum = "ERROR"
}
return sum
return sum, err
}
// HashLister does a md5sum equivalent for the hash type passed in
func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
sum := hashSum(ctx, ht, o)
sum, _ := hashSum(ctx, ht, o)
syncFprintf(w, "%*s %s\n", hash.Width(ht), sum, o.Remote())
})
}
// HashListerBase64 does a md5sum equivalent for the hash type passed in with base64 encoded
func HashListerBase64(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
sum, err := hashSum(ctx, ht, o)
if err == nil {
hexBytes, _ := hex.DecodeString(sum)
sum = base64.URLEncoding.EncodeToString(hexBytes)
}
width := base64.URLEncoding.EncodedLen(hash.Width(ht) / 2)
syncFprintf(w, "%*s %s\n", width, sum, o.Remote())
})
}
// Count counts the objects and their sizes in the Fs
//
// Obeys includes and excludes

37
fs/operations/operations_test.go
View File

@ -225,6 +225,43 @@ func TestHashSums(t *testing.T) {
!strings.Contains(res, " potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
// QuickXorHash Sum
buf.Reset()
var ht hash.Type
err = ht.Set("QuickXorHash")
require.NoError(t, err)
err = operations.HashLister(context.Background(), ht, r.Fremote, &buf)
require.NoError(t, err)
res = buf.String()
if !strings.Contains(res, "2d00000000000000000000000100000000000000 empty space\n") &&
!strings.Contains(res, " UNSUPPORTED empty space\n") &&
!strings.Contains(res, " empty space\n") {
t.Errorf("empty space missing: %q", res)
}
if !strings.Contains(res, "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n") &&
!strings.Contains(res, " UNSUPPORTED potato2\n") &&
!strings.Contains(res, " potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
// QuickXorHash Sum with Base64 Encoded
buf.Reset()
err = operations.HashListerBase64(context.Background(), ht, r.Fremote, &buf)
require.NoError(t, err)
res = buf.String()
if !strings.Contains(res, "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n") &&
!strings.Contains(res, " UNSUPPORTED empty space\n") &&
!strings.Contains(res, " empty space\n") {
t.Errorf("empty space missing: %q", res)
}
if !strings.Contains(res, "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n") &&
!strings.Contains(res, " UNSUPPORTED potato2\n") &&
!strings.Contains(res, " potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
}
func TestSuffixName(t *testing.T) {