diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index e04c1bc05..19c3aeaed 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -7,13 +7,20 @@ import ( "os" "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) +var ( + outputBase64 = false +) + func init() { cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.BoolVarP(cmdFlags, &outputBase64, "base64", "", outputBase64, "Output base64 encoded hashsum") } var commandDefinition = &cobra.Command{ @@ -55,6 +62,9 @@ Then } fsrc := cmd.NewFsSrc(args[1:]) cmd.Run(false, false, command, func() error { + if outputBase64 { + return operations.HashListerBase64(context.Background(), ht, fsrc, os.Stdout) + } return operations.HashLister(context.Background(), ht, fsrc, os.Stdout) }) return nil diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 40f837932..4db00723b 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -4,7 +4,9 @@ package operations import ( "bytes" "context" + "encoding/base64" "encoding/csv" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -1042,8 +1044,9 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error { } // hashSum returns the human readable hash for ht passed in. This may -// be UNSUPPORTED or ERROR. -func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string { +// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will +// return an error. +func hashSum(ctx context.Context, ht hash.Type, o fs.Object) (string, error) { var err error tr := accounting.Stats(ctx).NewCheckingTransfer(o) defer func() { @@ -1056,17 +1059,30 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string { fs.Debugf(o, "Failed to read %v: %v", ht, err) sum = "ERROR" } - return sum + return sum, err } // HashLister does a md5sum equivalent for the hash type passed in func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error { return ListFn(ctx, f, func(o fs.Object) { - sum := hashSum(ctx, ht, o) + sum, _ := hashSum(ctx, ht, o) syncFprintf(w, "%*s %s\n", hash.Width(ht), sum, o.Remote()) }) } +// HashListerBase64 does a md5sum equivalent for the hash type passed in with base64 encoded +func HashListerBase64(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error { + return ListFn(ctx, f, func(o fs.Object) { + sum, err := hashSum(ctx, ht, o) + if err == nil { + hexBytes, _ := hex.DecodeString(sum) + sum = base64.URLEncoding.EncodeToString(hexBytes) + } + width := base64.URLEncoding.EncodedLen(hash.Width(ht) / 2) + syncFprintf(w, "%*s %s\n", width, sum, o.Remote()) + }) +} + // Count counts the objects and their sizes in the Fs // // Obeys includes and excludes diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 7d76485a7..abbafc55c 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -225,6 +225,43 @@ func TestHashSums(t *testing.T) { !strings.Contains(res, " potato2\n") { t.Errorf("potato2 missing: %q", res) } + + // QuickXorHash Sum + + buf.Reset() + var ht hash.Type + err = ht.Set("QuickXorHash") + require.NoError(t, err) + err = operations.HashLister(context.Background(), ht, r.Fremote, &buf) + require.NoError(t, err) + res = buf.String() + if !strings.Contains(res, "2d00000000000000000000000100000000000000 empty space\n") && + !strings.Contains(res, " UNSUPPORTED empty space\n") && + !strings.Contains(res, " empty space\n") { + t.Errorf("empty space missing: %q", res) + } + if !strings.Contains(res, "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n") && + !strings.Contains(res, " UNSUPPORTED potato2\n") && + !strings.Contains(res, " potato2\n") { + t.Errorf("potato2 missing: %q", res) + } + + // QuickXorHash Sum with Base64 Encoded + + buf.Reset() + err = operations.HashListerBase64(context.Background(), ht, r.Fremote, &buf) + require.NoError(t, err) + res = buf.String() + if !strings.Contains(res, "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n") && + !strings.Contains(res, " UNSUPPORTED empty space\n") && + !strings.Contains(res, " empty space\n") { + t.Errorf("empty space missing: %q", res) + } + if !strings.Contains(res, "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n") && + !strings.Contains(res, " UNSUPPORTED potato2\n") && + !strings.Contains(res, " potato2\n") { + t.Errorf("potato2 missing: %q", res) + } } func TestSuffixName(t *testing.T) {