yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove backend dependency from fs/hash

s3-about
Fionera 2019-09-23 15:32:36 +02:00 committed by Nick Craig-Wood
parent def411da62
commit 1dc8bcd48c
13 changed files with 138 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/dropbox/dropbox.go
View File

@ -39,6 +39,7 @@ import (
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/dropbox/dbhash"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
@ -105,10 +106,14 @@ var (
// A regexp matching path names for files Dropbox ignores
// See https://www.dropbox.com/en/help/145 - Ignored files
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
// DbHashType is the hash.Type for Dropbox
DbHashType hash.Type
)
// Register with Fs
func init() {
DbHashType = hash.RegisterHash("Dropbox", 64, dbhash.New)
fs.Register(&fs.RegInfo{
Name: "dropbox",
Description: "Dropbox",
@ -881,7 +886,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.Dropbox)
return hash.Set(DbHashType)
}
// ------------------------------------------------------------
@ -906,7 +911,7 @@ func (o *Object) Remote() string {
// Hash returns the dropbox special hash
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.Dropbox {
if t != DbHashType {
return "", hash.ErrUnsupported
}
err := o.readMetaData()

2
backend/local/local.go
View File

@ -686,7 +686,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Supported
return hash.Supported()
}
// ------------------------------------------------------------

2
backend/local/local_internal_test.go
View File

@ -44,7 +44,7 @@ func TestUpdatingCheck(t *testing.T) {
require.NoError(t, err)
o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
wrappedFd := readers.NewLimitedReadCloser(fd, -1)
hash, err := hash.NewMultiHasherTypes(hash.Supported)
hash, err := hash.NewMultiHasherTypes(hash.Supported())
require.NoError(t, err)
in := localOpenFile{
o: o,

14
backend/mailru/mailru.go
View File

@ -59,6 +59,9 @@ var (
ErrorDirAlreadyExists = errors.New("directory already exists")
ErrorDirSourceNotExists = errors.New("directory source does not exist")
ErrorInvalidName = errors.New("invalid characters in object name")
// MrHashType is the hash.Type for Mailru
MrHashType hash.Type
)
// Description of how to authorize
@ -74,6 +77,7 @@ var oauthConfig = &oauth2.Config{
// Register with Fs
func init() {
MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
fs.Register(&fs.RegInfo{
Name: "mailru",
Description: "Mail.ru Cloud",
@ -1591,7 +1595,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Skip an extra speedup request if file fits in hash.
if size > mrhash.Size {
// Request hash from source.
if srcHash, err := src.Hash(ctx, hash.Mailru); err == nil && srcHash != "" {
if srcHash, err := src.Hash(ctx, MrHashType); err == nil && srcHash != "" {
fileHash, _ = mrhash.DecodeString(srcHash)
}
@ -1762,7 +1766,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
hashType := hash.SHA1
// Calculate Mailru and spool verification hashes in transit
hashSet := hash.NewHashSet(hash.Mailru, hashType)
hashSet := hash.NewHashSet(MrHashType, hashType)
hasher, err := hash.NewMultiHasherTypes(hashSet)
if err != nil {
return nil, nil, err
@ -1784,7 +1788,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
return nil, nil, mrhash.ErrorInvalidHash
}
mrHash, err = mrhash.DecodeString(sums[hash.Mailru])
mrHash, err = mrhash.DecodeString(sums[MrHashType])
return
}
@ -1972,7 +1976,7 @@ func (o *Object) Size() int64 {
// Hash returns the MD5 or SHA1 sum of an object
// returning a lowercase hex string
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t == hash.Mailru {
if t == MrHashType {
return hex.EncodeToString(o.mrHash), nil
}
return "", hash.ErrUnsupported
@ -2354,7 +2358,7 @@ func (f *Fs) Precision() time.Duration {
// Hashes returns the supported hash sets
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.Mailru)
return hash.Set(MrHashType)
}
// Features returns the optional features of this Fs

9
backend/onedrive/onedrive.go
View File

@ -17,6 +17,7 @@ import (
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/onedrive/api"
"github.com/rclone/rclone/backend/onedrive/quickxorhash"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
@ -65,10 +66,14 @@ var (
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
RedirectURL: oauthutil.RedirectLocalhostURL,
}
// QuickXorHashType is the hash.Type for OneDrive
QuickXorHashType hash.Type
)
// Register with Fs
func init() {
QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
fs.Register(&fs.RegInfo{
Name: "onedrive",
Description: "Microsoft OneDrive",
@ -1194,7 +1199,7 @@ func (f *Fs) Hashes() hash.Set {
if f.driveType == driveTypePersonal {
return hash.Set(hash.SHA1)
}
return hash.Set(hash.QuickXorHash)
return hash.Set(QuickXorHashType)
}
// PublicLink returns a link for downloading without accout.
@ -1270,7 +1275,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
return o.sha1, nil
}
} else {
if t == hash.QuickXorHash {
if t == QuickXorHashType {
return o.quickxorhash, nil
}
}

5
cmd/dbhashsum/dbhashsum.go
View File

@ -4,7 +4,9 @@ import (
"context"
"os"
"github.com/rclone/rclone/backend/dropbox/dbhash"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
@ -26,7 +28,8 @@ The output is in the same format as md5sum and sha1sum.
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
return operations.DropboxHashSum(context.Background(), fsrc, os.Stdout)
dbHashType := hash.RegisterHash("Dropbox", 64, dbhash.New)
return operations.HashLister(context.Background(), dbHashType, fsrc, os.Stdout)
})
},
}

2
cmd/hashsum/hashsum.go
View File

@ -41,7 +41,7 @@ Then
cmd.CheckArgs(0, 2, command, args)
if len(args) == 0 {
fmt.Printf("Supported hashes are:\n")
for _, ht := range hash.Supported.Array() {
for _, ht := range hash.Supported().Array() {
fmt.Printf(" * %v\n", ht)
}
return nil

178
fs/hash/hash.go
View File

@ -12,64 +12,88 @@ import (
"github.com/jzelinskie/whirlpool"
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/dropbox/dbhash"
"github.com/rclone/rclone/backend/mailru/mrhash"
"github.com/rclone/rclone/backend/onedrive/quickxorhash"
)
// Type indicates a standard hashing algorithm
type Type int
type hashDefinition struct {
width int
name string
newFunc func() hash.Hash
hashType Type
}
var hashes []*hashDefinition
var highestType Type = 1
// RegisterHash adds a new Hash to the list and returns it Type
func RegisterHash(name string, width int, newFunc func() hash.Hash) Type {
definition := &hashDefinition{
name: name,
width: width,
newFunc: newFunc,
hashType: highestType,
}
hashes = append(hashes, definition)
highestType = highestType << 1
return definition.hashType
}
// ErrUnsupported should be returned by filesystem,
// if it is requested to deliver an unsupported hash type.
var ErrUnsupported = errors.New("hash type not supported")
const (
var (
// None indicates no hashes are supported
None Type
// MD5 indicates MD5 support
MD5 Type = 1 << iota
MD5 Type
// SHA1 indicates SHA-1 support
SHA1
// Dropbox indicates Dropbox special hash
// https://www.dropbox.com/developers/reference/content-hash
Dropbox
// QuickXorHash indicates Microsoft onedrive hash
// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
QuickXorHash
SHA1 Type
// Whirlpool indicates Whirlpool support
Whirlpool
Whirlpool Type
// CRC32 indicates CRC-32 support
CRC32
// Mailru indicates Mailru special hash
Mailru
// None indicates no hashes are supported
None Type = 0
CRC32 Type
)
func init() {
MD5 = RegisterHash("MD5", 32, md5.New)
SHA1 = RegisterHash("SHA-1", 40, sha1.New)
Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New)
CRC32 = RegisterHash("CRC32", 8, func() hash.Hash { return crc32.NewIEEE() })
}
// Supported returns a set of all the supported hashes by
// HashStream and MultiHasher.
var Supported = NewHashSet(MD5, SHA1, Dropbox, QuickXorHash, Whirlpool, CRC32, Mailru)
func Supported() Set {
var types []Type
for _, v := range hashes {
types = append(types, v.hashType)
}
return NewHashSet(types...)
}
// Width returns the width in characters for any HashType
var Width = map[Type]int{
MD5: 32,
SHA1: 40,
Dropbox: 64,
QuickXorHash: 40,
Whirlpool: 128,
CRC32: 8,
Mailru: 40,
func Width(hashType Type) int {
for _, v := range hashes {
if v.hashType == hashType {
return v.width
}
}
return 0
}
// Stream will calculate hashes of all supported hash types.
func Stream(r io.Reader) (map[Type]string, error) {
return StreamTypes(r, Supported)
return StreamTypes(r, Supported())
}
// StreamTypes will calculate hashes of the requested hash types.
@ -93,52 +117,34 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
// String returns a string representation of the hash type.
// The function will panic if the hash type is unknown.
func (h Type) String() string {
switch h {
case None:
if h == None {
return "None"
case MD5:
return "MD5"
case SHA1:
return "SHA-1"
case Dropbox:
return "DropboxHash"
case QuickXorHash:
return "QuickXorHash"
case Whirlpool:
return "Whirlpool"
case CRC32:
return "CRC-32"
case Mailru:
return "MailruHash"
default:
err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
panic(err)
}
for _, v := range hashes {
if v.hashType == h {
return v.name
}
}
err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
panic(err)
}
// Set a Type from a flag
func (h *Type) Set(s string) error {
switch s {
case "None":
if s == "None" {
*h = None
case "MD5":
*h = MD5
case "SHA-1":
*h = SHA1
case "DropboxHash":
*h = Dropbox
case "QuickXorHash":
*h = QuickXorHash
case "Whirlpool":
*h = Whirlpool
case "CRC-32":
*h = CRC32
case "MailruHash":
*h = Mailru
default:
return errors.Errorf("Unknown hash type %q", s)
}
return nil
for _, v := range hashes {
if v.name == s {
*h = v.hashType
return nil
}
}
return errors.Errorf("Unknown hash type %q", s)
}
// Type of the value
@ -150,32 +156,28 @@ func (h Type) Type() string {
// The types must be a subset of SupportedHashes,
// and this function must support all types.
func fromTypes(set Set) (map[Type]hash.Hash, error) {
if !set.SubsetOf(Supported) {
if !set.SubsetOf(Supported()) {
return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
}
var hashers = make(map[Type]hash.Hash)
types := set.Array()
for _, t := range types {
switch t {
case MD5:
hashers[t] = md5.New()
case SHA1:
hashers[t] = sha1.New()
case Dropbox:
hashers[t] = dbhash.New()
case QuickXorHash:
hashers[t] = quickxorhash.New()
case Whirlpool:
hashers[t] = whirlpool.New()
case CRC32:
hashers[t] = crc32.NewIEEE()
case Mailru:
hashers[t] = mrhash.New()
default:
for _, v := range hashes {
if t != v.hashType {
continue
}
hashers[t] = v.newFunc()
break
}
if hashers[t] == nil {
err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
panic(err)
}
}
return hashers, nil
}
@ -202,7 +204,7 @@ type MultiHasher struct {
// NewMultiHasher will return a hash writer that will write all
// supported hash types.
func NewMultiHasher() *MultiHasher {
h, err := NewMultiHasherTypes(Supported)
h, err := NewMultiHasherTypes(Supported())
if err != nil {
panic("internal error: could not create multihasher")
}

34
fs/hash/hash_test.go
View File

@ -3,6 +3,7 @@ package hash_test
import (
"bytes"
"io"
"log"
"testing"
"github.com/rclone/rclone/fs/hash"
@ -23,6 +24,7 @@ func TestHashSet(t *testing.T) {
assert.Len(t, a, 0)
h = h.Add(hash.MD5)
log.Println(h)
assert.Equal(t, 1, h.Count())
assert.Equal(t, hash.MD5, h.GetOne())
a = h.Array()
@ -30,10 +32,10 @@ func TestHashSet(t *testing.T) {
assert.Equal(t, a[0], hash.MD5)
// Test overlap, with all hashes
h = h.Overlap(hash.Supported)
h = h.Overlap(hash.Supported())
assert.Equal(t, 1, h.Count())
assert.Equal(t, hash.MD5, h.GetOne())
assert.True(t, h.SubsetOf(hash.Supported))
assert.True(t, h.SubsetOf(hash.Supported()))
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
h = h.Add(hash.SHA1)
@ -42,7 +44,7 @@ func TestHashSet(t *testing.T) {
if !(one == hash.MD5 || one == hash.SHA1) {
t.Fatalf("expected to be either MD5 or SHA1, got %v", one)
}
assert.True(t, h.SubsetOf(hash.Supported))
assert.True(t, h.SubsetOf(hash.Supported()))
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.MD5)))
assert.False(t, h.SubsetOf(hash.NewHashSet(hash.SHA1)))
assert.True(t, h.SubsetOf(hash.NewHashSet(hash.MD5, hash.SHA1)))
@ -69,26 +71,20 @@ var hashTestSet = []hashTest{
{
input: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
output: map[hash.Type]string{
hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe",
hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166",
hash.Dropbox: "214d2fcf3566e94c99ad2f59bd993daca46d8521a0c447adf4b324f53fddc0c7",
hash.QuickXorHash: "0110c000085000031c0001095ec00218d0000700",
hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4",
hash.CRC32: "a6041d7e",
hash.Mailru: "0102030405060708090a0b0c0d0e000000000000",
hash.MD5: "bf13fc19e5151ac57d4252e0e0f87abe",
hash.SHA1: "3ab6543c08a75f292a5ecedac87ec41642d12166",
hash.Whirlpool: "eddf52133d4566d763f716e853d6e4efbabd29e2c2e63f56747b1596172851d34c2df9944beb6640dbdbe3d9b4eb61180720a79e3d15baff31c91e43d63869a4",
hash.CRC32: "a6041d7e",
},
},
// Empty data set
{
input: []byte{},
output: map[hash.Type]string{
hash.MD5: "d41d8cd98f00b204e9800998ecf8427e",
hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
hash.Dropbox: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
hash.QuickXorHash: "0000000000000000000000000000000000000000",
hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
hash.CRC32: "00000000",
hash.Mailru: "0000000000000000000000000000000000000000",
hash.MD5: "d41d8cd98f00b204e9800998ecf8427e",
hash.SHA1: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
hash.Whirlpool: "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3",
hash.CRC32: "00000000",
},
},
}
@ -159,8 +155,8 @@ func TestHashStreamTypes(t *testing.T) {
}
func TestHashSetStringer(t *testing.T) {
h := hash.NewHashSet(hash.SHA1, hash.MD5, hash.Dropbox, hash.QuickXorHash)
assert.Equal(t, h.String(), "[MD5, SHA-1, DropboxHash, QuickXorHash]")
h := hash.NewHashSet(hash.SHA1, hash.MD5)
assert.Equal(t, h.String(), "[MD5, SHA-1]")
h = hash.NewHashSet(hash.SHA1)
assert.Equal(t, h.String(), "[SHA-1]")
h = hash.NewHashSet()

2
fs/object/object.go
View File

@ -79,7 +79,7 @@ func (memoryFs) String() string { return "memory" }
func (memoryFs) Precision() time.Duration { return time.Nanosecond }
// Returns the supported hash types of the filesystem
func (memoryFs) Hashes() hash.Set { return hash.Supported }
func (memoryFs) Hashes() hash.Set { return hash.Supported() }
// Features returns the optional features of this Fs
func (memoryFs) Features() *fs.Features { return &fs.Features{} }

2
fs/object/object_test.go
View File

@ -53,7 +53,7 @@ func TestMemoryFs(t *testing.T) {
assert.Equal(t, "", f.Root())
assert.Equal(t, "memory", f.String())
assert.Equal(t, time.Nanosecond, f.Precision())
assert.Equal(t, hash.Supported, f.Hashes())
assert.Equal(t, hash.Supported(), f.Hashes())
assert.Equal(t, &fs.Features{}, f.Features())
entries, err := f.List(context.Background(), "")

11
fs/operations/operations.go
View File

@ -980,15 +980,6 @@ func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error {
return HashLister(ctx, hash.SHA1, f, w)
}
// DropboxHashSum list the Fs to the supplied writer
//
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func DropboxHashSum(ctx context.Context, f fs.Fs, w io.Writer) error {
return HashLister(ctx, hash.Dropbox, f, w)
}
// hashSum returns the human readable hash for ht passed in. This may
// be UNSUPPORTED or ERROR.
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
@ -1011,7 +1002,7 @@ func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
sum := hashSum(ctx, ht, o)
syncFprintf(w, "%*s %s\n", hash.Width[ht], sum, o.Remote())
syncFprintf(w, "%*s %s\n", hash.Width(ht), sum, o.Remote())
})
}

18
fs/operations/operations_test.go
View File

@ -225,23 +225,6 @@ func TestHashSums(t *testing.T) {
!strings.Contains(res, " potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
// Dropbox Hash Sum
buf.Reset()
err = operations.DropboxHashSum(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res = buf.String()
if !strings.Contains(res, "fc62b10ec59efa8041f5a6c924d7c91572c1bbda280d9e01312b660804df1d47 empty space\n") &&
!strings.Contains(res, " UNSUPPORTED empty space\n") &&
!strings.Contains(res, " empty space\n") {
t.Errorf("empty space missing: %q", res)
}
if !strings.Contains(res, "a979481df794fed9c3990a6a422e0b1044ac802c15fab13af9c687f8bdbee01a potato2\n") &&
!strings.Contains(res, " UNSUPPORTED potato2\n") &&
!strings.Contains(res, " potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
}
func TestSuffixName(t *testing.T) {
@ -1260,7 +1243,6 @@ func TestListFormat(t *testing.T) {
}{
{hash.MD5, "0cc175b9c0f1b6a831c399e269772661"},
{hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"},
{hash.Dropbox, "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8"},
} {
list.SetOutput(nil)
list.AddHash(test.ht)