From d50572b1083b72deba6a6337f41643371f16f002 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 17 Jan 2024 11:47:33 +0000 Subject: [PATCH] operations: add operations/hashsum to the rc as rclone hashsum equivalent Fixes #7569 --- fs/operations/rc.go | 77 +++++++++++++++++++++++++++++++++++ fs/operations/rc_test.go | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 533c3601e..1435e66de 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -876,3 +876,80 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { } return out, nil } + +func init() { + rc.Add(rc.Call{ + Path: "operations/hashsum", + AuthRequired: true, + Fn: rcHashsum, + Title: "Produces a hashsum file for all the objects in the path.", + Help: `Produces a hash file for all the objects in the path using the hash +named. The output is in the same format as the standard +md5sum/sha1sum tool. + +This takes the following parameters: + +- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem + - this can point to a file and just that file will be returned in the listing. +- hashType - type of hash to be used +- download - check by downloading rather than with hash (boolean) +- base64 - output the hashes in base64 rather than hex (boolean) + +If you supply the download flag, it will download the data from the +remote and create the hash on the fly. This can be useful for remotes +that don't support the given hash or if you really want to check all +the data. + +Note that if you wish to supply a checkfile to check hashes against +the current files then you should use operations/check instead of +operations/hashsum. + +Returns: + +- hashsum - array of strings of the hashes +- hashType - type of hash used + +Example: + + $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true + { + "hashType": "md5", + "hashsum": [ + "WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh", + "v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh", + "VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh", + ] + } + +See the [hashsum](/commands/rclone_hashsum/) command for more information on the above. +`, + }) +} + +// Hashsum a directory +func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) { + ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs") + if err != nil { + return nil, err + } + + download, _ := in.GetBool("download") + base64, _ := in.GetBool("base64") + hashType, err := in.GetString("hashType") + if err != nil { + return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err) + } + var ht hash.Type + err = ht.Set(hashType) + if err != nil { + return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err) + } + + hashes := []string{} + err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes}) + out = rc.Params{ + "hashType": ht.String(), + "hashsum": hashes, + } + return out, err +} diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index aec817b18..990cb9c6e 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -2,6 +2,7 @@ package operations_test import ( "context" + "fmt" "net/http" "net/http/httptest" "net/url" @@ -14,6 +15,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fstest" @@ -779,3 +781,88 @@ deadbeefcafe00000000000000000000 subdir/file2 }) } + +// operations/hashsum: hashsum a directory +func TestRcHashsum(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/hashsum") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "file1 contents" + file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + hasher := hash.NewMultiHasher() + _, err := hasher.Write([]byte(file1Contents)) + require.NoError(t, err) + + for _, test := range []struct { + ht hash.Type + base64 bool + download bool + }{ + { + ht: r.Fremote.Hashes().GetOne(), + }, { + ht: r.Fremote.Hashes().GetOne(), + base64: true, + }, { + ht: hash.Whirlpool, + base64: false, + download: true, + }, { + ht: hash.Whirlpool, + base64: true, + download: true, + }, + } { + t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) { + file1Hash, err := hasher.SumString(test.ht, test.base64) + require.NoError(t, err) + + in := rc.Params{ + "fs": r.FremoteName, + "hashType": test.ht.String(), + "base64": test.base64, + "download": test.download, + } + + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, test.ht.String(), out["hashType"]) + want := []string{ + fmt.Sprintf("%s hashsum-file1", file1Hash), + } + assert.Equal(t, want, out["hashsum"]) + }) + } +} + +// operations/hashsum: hashsum a single file +func TestRcHashsumFile(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/hashsum") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "file1 contents" + file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) + file2Contents := "file2 contents" + file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + + // Make an fs pointing to just the file + fsString := path.Join(r.FremoteName, file1.Path) + + in := rc.Params{ + "fs": fsString, + "hashType": "MD5", + "download": true, + } + + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "md5", out["hashType"]) + assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"]) +}