From 3a04d0d1a906692600491c31455d5244f35660a4 Mon Sep 17 00:00:00 2001 From: buengese Date: Mon, 5 Aug 2019 13:43:39 +0200 Subject: [PATCH] march: rework testcases to better reflect real use --- fs/march/march_test.go | 266 +++++++++++++++++++++++++++++++++++++++++ fstest/fstest.go | 43 +++++++ 2 files changed, 309 insertions(+) diff --git a/fs/march/march_test.go b/fs/march/march_test.go index b1f11067c..eae1ea86a 100644 --- a/fs/march/march_test.go +++ b/fs/march/march_test.go @@ -3,16 +3,282 @@ package march import ( + "context" + "errors" "fmt" "strings" + "sync" "testing" + _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/mockdir" "github.com/rclone/rclone/fstest/mockobject" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// Some times used in the tests +var ( + t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") +) + +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +type marchTester struct { + ctx context.Context // internal context for controlling go-routines + cancel func() // cancel the context + srcOnly fs.DirEntries + dstOnly fs.DirEntries + match fs.DirEntries + entryMutex sync.Mutex + errorMu sync.Mutex // Mutex covering the error variables + err error + noRetryErr error + fatalErr error + noTraverse bool +} + +// DstOnly have an object which is in the destination only +func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) { + mt.entryMutex.Lock() + mt.dstOnly = append(mt.dstOnly, dst) + mt.entryMutex.Unlock() + + switch dst.(type) { + case fs.Object: + return false + case fs.Directory: + return true + default: + panic("Bad object in DirEntries") + } +} + +// SrcOnly have an object which is in the source only +func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) { + mt.entryMutex.Lock() + mt.srcOnly = append(mt.srcOnly, src) + mt.entryMutex.Unlock() + + switch src.(type) { + case fs.Object: + return false + case fs.Directory: + return true + default: + panic("Bad object in DirEntries") + } +} + +// Match is called when src and dst are present, so sync src to dst +func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) { + mt.entryMutex.Lock() + mt.match = append(mt.match, src) + mt.entryMutex.Unlock() + + switch src.(type) { + case fs.Object: + return false + case fs.Directory: + // Do the same thing to the entire contents of the directory + _, ok := dst.(fs.Directory) + if ok { + return true + } + // FIXME src is dir, dst is file + err := errors.New("can't overwrite file with directory") + fs.Errorf(dst, "%v", err) + mt.processError(err) + default: + panic("Bad object in DirEntries") + } + return false +} + +func (mt *marchTester) processError(err error) { + if err == nil { + return + } + mt.errorMu.Lock() + defer mt.errorMu.Unlock() + switch { + case fserrors.IsFatalError(err): + if !mt.aborting() { + fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err) + mt.cancel() + } + mt.fatalErr = err + case fserrors.IsNoRetryError(err): + mt.noRetryErr = err + default: + mt.err = err + } +} + +func (mt *marchTester) currentError() error { + mt.errorMu.Lock() + defer mt.errorMu.Unlock() + if mt.fatalErr != nil { + return mt.fatalErr + } + if mt.err != nil { + return mt.err + } + return mt.noRetryErr +} + +func (mt *marchTester) aborting() bool { + return mt.ctx.Err() != nil +} + +func TestMarch(t *testing.T) { + for _, test := range []struct { + what string + fileSrcOnly []string + dirSrcOnly []string + fileDstOnly []string + dirDstOnly []string + fileMatch []string + dirMatch []string + }{ + { + what: "source only", + fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, + dirSrcOnly: []string{"sub dir"}, + }, + { + what: "identical", + fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, + dirMatch: []string{"sub dir", "sub dir/sub sub dir"}, + }, + { + what: "typical sync", + fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, + dirSrcOnly: []string{"srcOnlyDir"}, + fileMatch: []string{"match", "matchDir/match file"}, + dirMatch: []string{"matchDir"}, + fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"}, + dirDstOnly: []string{"dstOnlyDir"}, + }, + } { + t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + var srcOnly []fstest.Item + var dstOnly []fstest.Item + var match []fstest.Item + + ctx, cancel := context.WithCancel(context.Background()) + + for _, f := range test.fileSrcOnly { + srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) + } + for _, f := range test.fileDstOnly { + dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1)) + } + for _, f := range test.fileMatch { + match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) + } + + mt := &marchTester{ + ctx: ctx, + cancel: cancel, + noTraverse: false, + } + m := &March{ + Ctx: ctx, + Fdst: r.Fremote, + Fsrc: r.Flocal, + Dir: "", + NoTraverse: mt.noTraverse, + Callback: mt, + DstIncludeAll: filter.Active.Opt.DeleteExcluded, + } + + mt.processError(m.Run()) + mt.cancel() + err := mt.currentError() + require.NoError(t, err) + + fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, "srcOnly") + fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, "dstOnly") + fstest.CompareItems(t, mt.match, match, test.dirMatch, "match") + }) + } +} + +func TestMarchNoTraverse(t *testing.T) { + for _, test := range []struct { + what string + fileSrcOnly []string + dirSrcOnly []string + fileMatch []string + dirMatch []string + }{ + { + what: "source only", + fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, + dirSrcOnly: []string{"sub dir"}, + }, + { + what: "identical", + fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, + }, + { + what: "typical sync", + fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, + fileMatch: []string{"match", "matchDir/match file"}, + }, + } { + t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + var srcOnly []fstest.Item + var match []fstest.Item + + ctx, cancel := context.WithCancel(context.Background()) + + for _, f := range test.fileSrcOnly { + srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) + } + for _, f := range test.fileMatch { + match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) + } + + mt := &marchTester{ + ctx: ctx, + cancel: cancel, + noTraverse: true, + } + m := &March{ + Ctx: ctx, + Fdst: r.Fremote, + Fsrc: r.Flocal, + Dir: "", + NoTraverse: mt.noTraverse, + Callback: mt, + DstIncludeAll: filter.Active.Opt.DeleteExcluded, + } + + mt.processError(m.Run()) + mt.cancel() + err := mt.currentError() + require.NoError(t, err) + + fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, "srcOnly") + fstest.CompareItems(t, mt.match, match, test.dirMatch, "match") + }) + } +} + func TestNewMatchEntries(t *testing.T) { var ( a = mockobject.Object("path/a") diff --git a/fstest/fstest.go b/fstest/fstest.go index c2b2b2672..411b4c3cc 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -349,6 +349,49 @@ func CheckItems(t *testing.T, f fs.Fs, items ...Item) { CheckListingWithPrecision(t, f, items, nil, fs.GetModifyWindow(f)) } +// CompareItems compares a set of DirEntries to a slice of items and a lit of dirs +func CompareItems(t *testing.T, entries fs.DirEntries, items []Item, expectedDirs []string, what string) { + is := NewItems(items) + precision, _ := time.ParseDuration("1s") + var objs []fs.Object + var dirs []fs.Directory + wantListing1, wantListing2 := makeListingFromItems(items) + for _, entry := range entries { + switch x := entry.(type) { + case fs.Directory: + dirs = append(dirs, x) + case fs.Object: + objs = append(objs, x) + // do nothing + default: + t.Fatalf("unknown object type %T", entry) + } + } + + gotListing := makeListingFromObjects(objs) + listingOK := wantListing1 == gotListing || wantListing2 == gotListing + assert.True(t, listingOK, fmt.Sprintf("%s not equal, want\n %s or\n %s got\n %s", what, wantListing1, wantListing2, gotListing)) + for _, obj := range objs { + require.NotNil(t, obj) + is.Find(t, obj, precision) + } + is.Done(t) + // Check the directories + if expectedDirs != nil { + expectedDirsCopy := make([]string, len(expectedDirs)) + for i, dir := range expectedDirs { + expectedDirsCopy[i] = WinPath(Normalize(dir)) + } + actualDirs := []string{} + for _, dir := range dirs { + actualDirs = append(actualDirs, WinPath(Normalize(dir.Remote()))) + } + sort.Strings(actualDirs) + sort.Strings(expectedDirsCopy) + assert.Equal(t, expectedDirsCopy, actualDirs, "directories not equal") + } +} + // Time parses a time string or logs a fatal error func Time(timeString string) time.Time { t, err := time.Parse(time.RFC3339Nano, timeString)