diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 57ed90a83..9dbfa5c20 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -39,6 +39,7 @@ import ( "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team" "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users" "github.com/pkg/errors" + "github.com/rclone/rclone/backend/dropbox/dbhash" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" @@ -105,10 +106,14 @@ var ( // A regexp matching path names for files Dropbox ignores // See https://www.dropbox.com/en/help/145 - Ignored files ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`) + + // DbHashType is the hash.Type for Dropbox + DbHashType hash.Type ) // Register with Fs func init() { + DbHashType = hash.RegisterHash("Dropbox", 64, dbhash.New) fs.Register(&fs.RegInfo{ Name: "dropbox", Description: "Dropbox", @@ -881,7 +886,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { // Hashes returns the supported hash sets. func (f *Fs) Hashes() hash.Set { - return hash.Set(hash.Dropbox) + return hash.Set(DbHashType) } // ------------------------------------------------------------ @@ -906,7 +911,7 @@ func (o *Object) Remote() string { // Hash returns the dropbox special hash func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { - if t != hash.Dropbox { + if t != DbHashType { return "", hash.ErrUnsupported } err := o.readMetaData() diff --git a/backend/local/local.go b/backend/local/local.go index 0f40dfe8f..fe5f87a2c 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -686,7 +686,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // Hashes returns the supported hash sets. func (f *Fs) Hashes() hash.Set { - return hash.Supported + return hash.Supported() } // ------------------------------------------------------------ diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index ca3eee788..abdd74dad 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -44,7 +44,7 @@ func TestUpdatingCheck(t *testing.T) { require.NoError(t, err) o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}} wrappedFd := readers.NewLimitedReadCloser(fd, -1) - hash, err := hash.NewMultiHasherTypes(hash.Supported) + hash, err := hash.NewMultiHasherTypes(hash.Supported()) require.NoError(t, err) in := localOpenFile{ o: o, diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 31820ccc8..f799aec3e 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -59,6 +59,9 @@ var ( ErrorDirAlreadyExists = errors.New("directory already exists") ErrorDirSourceNotExists = errors.New("directory source does not exist") ErrorInvalidName = errors.New("invalid characters in object name") + + // MrHashType is the hash.Type for Mailru + MrHashType hash.Type ) // Description of how to authorize @@ -74,6 +77,7 @@ var oauthConfig = &oauth2.Config{ // Register with Fs func init() { + MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New) fs.Register(&fs.RegInfo{ Name: "mailru", Description: "Mail.ru Cloud", @@ -1591,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Skip an extra speedup request if file fits in hash. if size > mrhash.Size { // Request hash from source. - if srcHash, err := src.Hash(ctx, hash.Mailru); err == nil && srcHash != "" { + if srcHash, err := src.Hash(ctx, MrHashType); err == nil && srcHash != "" { fileHash, _ = mrhash.DecodeString(srcHash) } @@ -1762,7 +1766,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj hashType := hash.SHA1 // Calculate Mailru and spool verification hashes in transit - hashSet := hash.NewHashSet(hash.Mailru, hashType) + hashSet := hash.NewHashSet(MrHashType, hashType) hasher, err := hash.NewMultiHasherTypes(hashSet) if err != nil { return nil, nil, err @@ -1784,7 +1788,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj return nil, nil, mrhash.ErrorInvalidHash } - mrHash, err = mrhash.DecodeString(sums[hash.Mailru]) + mrHash, err = mrhash.DecodeString(sums[MrHashType]) return } @@ -1972,7 +1976,7 @@ func (o *Object) Size() int64 { // Hash returns the MD5 or SHA1 sum of an object // returning a lowercase hex string func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { - if t == hash.Mailru { + if t == MrHashType { return hex.EncodeToString(o.mrHash), nil } return "", hash.ErrUnsupported @@ -2354,7 +2358,7 @@ func (f *Fs) Precision() time.Duration { // Hashes returns the supported hash sets func (f *Fs) Hashes() hash.Set { - return hash.Set(hash.Mailru) + return hash.Set(MrHashType) } // Features returns the optional features of this Fs diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index c23af9b21..7ff407de4 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/rclone/rclone/backend/onedrive/api" + "github.com/rclone/rclone/backend/onedrive/quickxorhash" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" @@ -65,10 +66,14 @@ var ( ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, } + + // QuickXorHashType is the hash.Type for OneDrive + QuickXorHashType hash.Type ) // Register with Fs func init() { + QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New) fs.Register(&fs.RegInfo{ Name: "onedrive", Description: "Microsoft OneDrive", @@ -1194,7 +1199,7 @@ func (f *Fs) Hashes() hash.Set { if f.driveType == driveTypePersonal { return hash.Set(hash.SHA1) } - return hash.Set(hash.QuickXorHash) + return hash.Set(QuickXorHashType) } // PublicLink returns a link for downloading without accout. @@ -1270,7 +1275,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { return o.sha1, nil } } else { - if t == hash.QuickXorHash { + if t == QuickXorHashType { return o.quickxorhash, nil } } diff --git a/cmd/dbhashsum/dbhashsum.go b/cmd/dbhashsum/dbhashsum.go index f83313861..61715b377 100644 --- a/cmd/dbhashsum/dbhashsum.go +++ b/cmd/dbhashsum/dbhashsum.go @@ -4,7 +4,9 @@ import ( "context" "os" + "github.com/rclone/rclone/backend/dropbox/dbhash" "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) @@ -26,7 +28,8 @@ The output is in the same format as md5sum and sha1sum. cmd.CheckArgs(1, 1, command, args) fsrc := cmd.NewFsSrc(args) cmd.Run(false, false, command, func() error { - return operations.DropboxHashSum(context.Background(), fsrc, os.Stdout) + dbHashType := hash.RegisterHash("Dropbox", 64, dbhash.New) + return operations.HashLister(context.Background(), dbHashType, fsrc, os.Stdout) }) }, } diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index 91e260472..e04c1bc05 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -41,7 +41,7 @@ Then cmd.CheckArgs(0, 2, command, args) if len(args) == 0 { fmt.Printf("Supported hashes are:\n") - for _, ht := range hash.Supported.Array() { + for _, ht := range hash.Supported().Array() { fmt.Printf(" * %v\n", ht) } return nil diff --git a/fs/hash/hash.go b/fs/hash/hash.go index 39c8abfeb..feee5200c 100644 --- a/fs/hash/hash.go +++ b/fs/hash/hash.go @@ -12,64 +12,88 @@ import ( "github.com/jzelinskie/whirlpool" "github.com/pkg/errors" - "github.com/rclone/rclone/backend/dropbox/dbhash" - "github.com/rclone/rclone/backend/mailru/mrhash" - "github.com/rclone/rclone/backend/onedrive/quickxorhash" ) // Type indicates a standard hashing algorithm type Type int +type hashDefinition struct { + width int + name string + newFunc func() hash.Hash + hashType Type +} + +var hashes []*hashDefinition +var highestType Type = 1 + +// RegisterHash adds a new Hash to the list and returns it Type +func RegisterHash(name string, width int, newFunc func() hash.Hash) Type { + definition := &hashDefinition{ + name: name, + width: width, + newFunc: newFunc, + hashType: highestType, + } + hashes = append(hashes, definition) + highestType = highestType << 1 + + return definition.hashType +} + // ErrUnsupported should be returned by filesystem, // if it is requested to deliver an unsupported hash type. var ErrUnsupported = errors.New("hash type not supported") -const ( +var ( + // None indicates no hashes are supported + None Type + // MD5 indicates MD5 support - MD5 Type = 1 << iota + MD5 Type // SHA1 indicates SHA-1 support - SHA1 - - // Dropbox indicates Dropbox special hash - // https://www.dropbox.com/developers/reference/content-hash - Dropbox - - // QuickXorHash indicates Microsoft onedrive hash - // https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash - QuickXorHash + SHA1 Type // Whirlpool indicates Whirlpool support - Whirlpool + Whirlpool Type // CRC32 indicates CRC-32 support - CRC32 - - // Mailru indicates Mailru special hash - Mailru - - // None indicates no hashes are supported - None Type = 0 + CRC32 Type ) +func init() { + MD5 = RegisterHash("MD5", 32, md5.New) + SHA1 = RegisterHash("SHA-1", 40, sha1.New) + Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New) + CRC32 = RegisterHash("CRC32", 8, func() hash.Hash { return crc32.NewIEEE() }) +} + // Supported returns a set of all the supported hashes by // HashStream and MultiHasher. -var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash, Whirlpool, CRC32, Mailru) +func Supported() Set { + var types []Type + for _, v := range hashes { + types = append(types, v.hashType) + } + + return NewHashSet(types...) +} // Width returns the width in characters for any HashType -var Width = map[Type]int{ - MD5: 32, - SHA1: 40, - Dropbox: 64, - QuickXorHash: 40, - Whirlpool: 128, - CRC32: 8, - Mailru: 40, +func Width(hashType Type) int { + for _, v := range hashes { + if v.hashType == hashType { + return v.width + } + } + + return 0 } // Stream will calculate hashes of all supported hash types. func Stream(r io.Reader) (map[Type]string, error) { - return StreamTypes(r, Supported) + return StreamTypes(r, Supported()) } // StreamTypes will calculate hashes of the requested hash types. @@ -93,52 +117,34 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) { // String returns a string representation of the hash type. // The function will panic if the hash type is unknown. func (h Type) String() string { - switch h { - case None: + if h == None { return "None" - case MD5: - return "MD5" - case SHA1: - return "SHA-1" - case Dropbox: - return "DropboxHash" - case QuickXorHash: - return "QuickXorHash" - case Whirlpool: - return "Whirlpool" - case CRC32: - return "CRC-32" - case Mailru: - return "MailruHash" - default: - err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)) - panic(err) } + + for _, v := range hashes { + if v.hashType == h { + return v.name + } + } + + err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)) + panic(err) } // Set a Type from a flag func (h *Type) Set(s string) error { - switch s { - case "None": + if s == "None" { *h = None - case "MD5": - *h = MD5 - case "SHA-1": - *h = SHA1 - case "DropboxHash": - *h = Dropbox - case "QuickXorHash": - *h = QuickXorHash - case "Whirlpool": - *h = Whirlpool - case "CRC-32": - *h = CRC32 - case "MailruHash": - *h = Mailru - default: - return errors.Errorf("Unknown hash type %q", s) } - return nil + + for _, v := range hashes { + if v.name == s { + *h = v.hashType + return nil + } + } + + return errors.Errorf("Unknown hash type %q", s) } // Type of the value @@ -150,32 +156,28 @@ func (h Type) Type() string { // The types must be a subset of SupportedHashes, // and this function must support all types. func fromTypes(set Set) (map[Type]hash.Hash, error) { - if !set.SubsetOf(Supported) { + if !set.SubsetOf(Supported()) { return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set)) } var hashers = make(map[Type]hash.Hash) + types := set.Array() for _, t := range types { - switch t { - case MD5: - hashers[t] = md5.New() - case SHA1: - hashers[t] = sha1.New() - case Dropbox: - hashers[t] = dbhash.New() - case QuickXorHash: - hashers[t] = quickxorhash.New() - case Whirlpool: - hashers[t] = whirlpool.New() - case CRC32: - hashers[t] = crc32.NewIEEE() - case Mailru: - hashers[t] = mrhash.New() - default: + for _, v := range hashes { + if t != v.hashType { + continue + } + + hashers[t] = v.newFunc() + break + } + + if hashers[t] == nil { err := fmt.Sprintf("internal error: Unsupported hash type %v", t) panic(err) } } + return hashers, nil } @@ -202,7 +204,7 @@ type MultiHasher struct { // NewMultiHasher will return a hash writer that will write all // supported hash types. func NewMultiHasher() *MultiHasher { - h, err := NewMultiHasherTypes(Supported) + h, err := NewMultiHasherTypes(Supported()) if err != nil { panic("internal error: could not create multihasher") } diff --git a/fs/hash/hash_test.go b/fs/hash/hash_test.go index 8b149eed9..e7653dbb1 100644 --- a/fs/hash/hash_test.go +++ b/fs/hash/hash_test.go @@ -3,6 +3,7 @@ package hash_test import ( "bytes" "io" + "log" "testing" "github.com/rclone/rclone/fs/hash" @@ -23,6 +24,7 @@ func TestHashSet(t *testing.T) { assert.Len(t, a, 0) h = h.Add(hash.MD5) + log.Println(h) assert.Equal(t, 1, h.Count()) assert.Equal(t, hash.MD5, h.GetOne()) a = h.Array() @@ -30,10 +32,10 @@ func TestHashSet(t *testing.T) { assert.Equal(t, a[0], hash.MD5) // Test overlap, with all hashes - h = h.Overlap(hash.Supported) + h = h.Overlap(hash.Supported()) assert.Equal(t, 1, h.Count()) assert.Equal(t, hash.MD5, h.GetOne()) - assert.True(t, h.SubsetOf(hash.Supported)) + assert.True(t, h.SubsetOf(hash.Supported())) assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5))) h = h.Add(hash.SHA1) @@ -42,7 +44,7 @@ func TestHashSet(t *testing.T) { if !(one == hash.MD5 || one == hash.SHA1) { t.Fatalf("expected to be either MD5 or SHA1, got %v", one) } - assert.True(t, h.SubsetOf(hash.Supported)) + assert.True(t, h.SubsetOf(hash.Supported())) assert.False(t, h.SubsetOf(hash.NewHashSet(hash.MD5))) assert.False(t, h.SubsetOf(hash.NewHashSet(hash.SHA1))) assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5, hash.SHA1))) @@ -69,26 +71,20 @@ var hashTestSet = []hashTest{ { input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, output: map[hash.Type]string{ - hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe", - hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166", - hash.Dropbox: "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7", - hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700", - hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4", - hash.CRC32: "a6041d7e", - hash.Mailru: "0102030405060708090a0b0c0d0e000000000000", + hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe", + hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166", + hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4", + hash.CRC32: "a6041d7e", }, }, // Empty data set { input: []byte{}, output: map[hash.Type]string{ - hash.MD5: "d41d8cd98f00b204e9800998ecf8427e", - hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", - hash.Dropbox: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - hash.QuickXorHash: "0000000000000000000000000000000000000000", - hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", - hash.CRC32: "00000000", - hash.Mailru: "0000000000000000000000000000000000000000", + hash.MD5: "d41d8cd98f00b204e9800998ecf8427e", + hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709", + hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", + hash.CRC32: "00000000", }, }, } @@ -159,8 +155,8 @@ func TestHashStreamTypes(t *testing.T) { } func TestHashSetStringer(t *testing.T) { - h := hash.NewHashSet(hash.SHA1, hash.MD5, hash.Dropbox, hash.QuickXorHash) - assert.Equal(t, h.String(), "[MD5, SHA-1, DropboxHash, QuickXorHash]") + h := hash.NewHashSet(hash.SHA1, hash.MD5) + assert.Equal(t, h.String(), "[MD5, SHA-1]") h = hash.NewHashSet(hash.SHA1) assert.Equal(t, h.String(), "[SHA-1]") h = hash.NewHashSet() diff --git a/fs/object/object.go b/fs/object/object.go index 14b26daa2..c88b9da02 100644 --- a/fs/object/object.go +++ b/fs/object/object.go @@ -79,7 +79,7 @@ func (memoryFs) String() string { return "memory" } func (memoryFs) Precision() time.Duration { return time.Nanosecond } // Returns the supported hash types of the filesystem -func (memoryFs) Hashes() hash.Set { return hash.Supported } +func (memoryFs) Hashes() hash.Set { return hash.Supported() } // Features returns the optional features of this Fs func (memoryFs) Features() *fs.Features { return &fs.Features{} } diff --git a/fs/object/object_test.go b/fs/object/object_test.go index 5e4ae1c00..d5cdfe619 100644 --- a/fs/object/object_test.go +++ b/fs/object/object_test.go @@ -53,7 +53,7 @@ func TestMemoryFs(t *testing.T) { assert.Equal(t, "", f.Root()) assert.Equal(t, "memory", f.String()) assert.Equal(t, time.Nanosecond, f.Precision()) - assert.Equal(t, hash.Supported, f.Hashes()) + assert.Equal(t, hash.Supported(), f.Hashes()) assert.Equal(t, &fs.Features{}, f.Features()) entries, err := f.List(context.Background(), "") diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 30ba3cd50..d0181e578 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -980,15 +980,6 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error { return HashLister(ctx, hash.SHA1, f, w) } -// DropboxHashSum list the Fs to the supplied writer -// -// Obeys includes and excludes -// -// Lists in parallel which may get them out of order -func DropboxHashSum(ctx context.Context, f fs.Fs, w io.Writer) error { - return HashLister(ctx, hash.Dropbox, f, w) -} - // hashSum returns the human readable hash for ht passed in. This may // be UNSUPPORTED or ERROR. func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string { @@ -1011,7 +1002,7 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string { func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error { return ListFn(ctx, f, func(o fs.Object) { sum := hashSum(ctx, ht, o) - syncFprintf(w, "%*s %s\n", hash.Width[ht], sum, o.Remote()) + syncFprintf(w, "%*s %s\n", hash.Width(ht), sum, o.Remote()) }) } diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index b0112683f..8826f3767 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -225,23 +225,6 @@ func TestHashSums(t *testing.T) { !strings.Contains(res, " potato2\n") { t.Errorf("potato2 missing: %q", res) } - - // Dropbox Hash Sum - - buf.Reset() - err = operations.DropboxHashSum(context.Background(), r.Fremote, &buf) - require.NoError(t, err) - res = buf.String() - if !strings.Contains(res, "fc62b10ec59efa8041f5a6c924d7c91572c1bbda280d9e01312b660804df1d47 empty space\n") && - !strings.Contains(res, " UNSUPPORTED empty space\n") && - !strings.Contains(res, " empty space\n") { - t.Errorf("empty space missing: %q", res) - } - if !strings.Contains(res, "a979481df794fed9c3990a6a422e0b1044ac802c15fab13af9c687f8bdbee01a potato2\n") && - !strings.Contains(res, " UNSUPPORTED potato2\n") && - !strings.Contains(res, " potato2\n") { - t.Errorf("potato2 missing: %q", res) - } } func TestSuffixName(t *testing.T) { @@ -1260,7 +1243,6 @@ func TestListFormat(t *testing.T) { }{ {hash.MD5, "0cc175b9c0f1b6a831c399e269772661"}, {hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}, - {hash.Dropbox, "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8"}, } { list.SetOutput(nil) list.AddHash(test.ht)