From 2b3d13a8416f73635185487b32939e5c092375ac Mon Sep 17 00:00:00 2001 From: Franklyn Tackitt Date: Tue, 25 Feb 2020 10:03:21 -0700 Subject: [PATCH] fs: Use --cutoff-mode hard,soft,catious instead of 3 --max-transfer-mode flags Fixes #2672 --- docs/content/docs.md | 10 +++--- fs/accounting/accounting.go | 2 +- fs/accounting/accounting_test.go | 6 ++-- fs/config.go | 2 +- fs/config/configflags/configflags.go | 39 ++++++---------------- fs/cutoffmode.go | 49 ++++++++++++++++++++++++++++ fs/cutoffmode_test.go | 6 ++++ fs/maxtransfermode.go | 12 ------- fs/operations/operations.go | 13 +++----- fs/operations/operations_test.go | 14 ++++---- 10 files changed, 86 insertions(+), 67 deletions(-) create mode 100644 fs/cutoffmode.go create mode 100644 fs/cutoffmode_test.go delete mode 100644 fs/maxtransfermode.go diff --git a/docs/content/docs.md b/docs/content/docs.md index b9f3d626b..4a1c341ab 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -736,18 +736,18 @@ When the limit is reached all transfers will stop immediately. Rclone will exit with exit code 8 if the transfer limit is reached. -### --max-transfer-(hard,soft,cautious) ### +### --cutoff-mode=hard|soft|cautious ### This modifies the behavior of `--max-transfer` -Defaults to `--max-transfer-hard`. +Defaults to `--cutoff-mode=hard`. -Specifiying `--max-transfer-hard` will stop transferring immediately +Specifiying `--cutoff-mode=hard` will stop transferring immediately when Rclone reaches the limit. -Specifiying `--max-transfer-soft` will stop starting new transfers +Specifiying `--cutoff-mode=soft` will stop starting new transfers when Rclone reaches the limit. -Specifiying `--max-transfer-cautious` will try to prevent Rclone +Specifiying `--cutoff-mode=cautious` will try to prevent Rclone from reaching the limit. ### --modify-window=TIME ### diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go index dc34a7c94..eb280b0a9 100644 --- a/fs/accounting/accounting.go +++ b/fs/accounting/accounting.go @@ -63,7 +63,7 @@ func newAccountSizeName(stats *StatsInfo, in io.ReadCloser, size int64, name str lpTime: time.Now(), max: -1, } - if fs.Config.MaxTransferMode != fs.MaxTransferModeSoft { + if fs.Config.CutoffMode == fs.CutoffModeHard { acc.max = int64((fs.Config.MaxTransfer)) } go acc.averageLoop() diff --git a/fs/accounting/accounting_test.go b/fs/accounting/accounting_test.go index b3b702503..9a00f48dd 100644 --- a/fs/accounting/accounting_test.go +++ b/fs/accounting/accounting_test.go @@ -197,12 +197,12 @@ func TestAccountAccounter(t *testing.T) { func TestAccountMaxTransfer(t *testing.T) { old := fs.Config.MaxTransfer - oldMode := fs.Config.MaxTransferMode + oldMode := fs.Config.CutoffMode fs.Config.MaxTransfer = 15 defer func() { fs.Config.MaxTransfer = old - fs.Config.MaxTransferMode = oldMode + fs.Config.CutoffMode = oldMode }() in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100))) @@ -222,7 +222,7 @@ func TestAccountMaxTransfer(t *testing.T) { assert.Equal(t, ErrorMaxTransferLimitReached, err) assert.True(t, fserrors.IsFatalError(err)) - fs.Config.MaxTransferMode = fs.MaxTransferModeSoft + fs.Config.CutoffMode = fs.CutoffModeSoft stats = NewStats() acc = newAccountSizeName(stats, in, 1, "test") diff --git a/fs/config.go b/fs/config.go index c7396b7bd..1bc6457bf 100644 --- a/fs/config.go +++ b/fs/config.go @@ -93,7 +93,7 @@ type ConfigInfo struct { UseServerModTime bool MaxTransfer SizeSuffix MaxDuration time.Duration - MaxTransferMode MaxTransferMode + CutoffMode CutoffMode MaxBacklog int MaxStatsGroups int StatsOneLine bool diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 454c1604b..e87d0d8ae 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -20,18 +20,15 @@ import ( var ( // these will get interpreted into fs.Config via SetFlags() below - verbose int - quiet bool - dumpHeaders bool - dumpBodies bool - deleteBefore bool - deleteDuring bool - deleteAfter bool - bindAddr string - disableFeatures string - maxTransferHard bool - maxTransferSoft bool - maxTransferCautious bool + verbose int + quiet bool + dumpHeaders bool + dumpBodies bool + deleteBefore bool + deleteDuring bool + deleteAfter bool + bindAddr string + disableFeatures string ) // AddFlags adds the non filing system specific flags to the command @@ -97,9 +94,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList) flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.") flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.") - flags.BoolVarP(flagSet, &maxTransferHard, "max-transfer-hard", "", false, "When transferring, stop immediately when --max-transfer is reached") - flags.BoolVarP(flagSet, &maxTransferSoft, "max-transfer-soft", "", false, "When transferring, stop starting new transfers when --max-transfer is reached") - flags.BoolVarP(flagSet, &maxTransferCautious, "max-transfer-cautious", "", false, "When transferring, try to avoid reaching --max-transfer") + flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS") flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.") flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.") flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.") @@ -184,20 +179,6 @@ func SetFlags() { fs.Config.DeleteMode = fs.DeleteModeDefault } - switch { - case maxTransferHard && (maxTransferSoft || maxTransferCautious), - maxTransferSoft && maxTransferCautious: - log.Fatalf(`Only one of --max-trans fer-hard, --max-transfer-soft or --max-transfer-cautious can be used.`) - case maxTransferHard: - fs.Config.MaxTransferMode = fs.MaxTransferModeHard - case maxTransferSoft: - fs.Config.MaxTransferMode = fs.MaxTransferModeSoft - case maxTransferCautious: - fs.Config.MaxTransferMode = fs.MaxTransferModeCautious - default: - fs.Config.MaxTransferMode = fs.MaxTransferModeDefault - } - if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" { log.Fatalf(`Can't use --compare-dest with --copy-dest.`) } diff --git a/fs/cutoffmode.go b/fs/cutoffmode.go new file mode 100644 index 000000000..19ec2b0c4 --- /dev/null +++ b/fs/cutoffmode.go @@ -0,0 +1,49 @@ +package fs + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" +) + +// CutoffMode describes the possible delete modes in the config +type CutoffMode byte + +// MaxTransferMode constants +const ( + CutoffModeHard CutoffMode = iota + CutoffModeSoft + CutoffModeCautious + CutoffModeDefault = CutoffModeHard +) + +var cutoffModeToString = []string{ + CutoffModeHard: "HARD", + CutoffModeSoft: "SOFT", + CutoffModeCautious: "CAUTIOUS", +} + +// String turns a LogLevel into a string +func (m CutoffMode) String() string { + if m >= CutoffMode(len(cutoffModeToString)) { + return fmt.Sprintf("CutoffMode(%d)", m) + } + return cutoffModeToString[m] +} + +// Set a LogLevel +func (m *CutoffMode) Set(s string) error { + for n, name := range cutoffModeToString { + if s != "" && name == strings.ToUpper(s) { + *m = CutoffMode(n) + return nil + } + } + return errors.Errorf("Unknown cutoff mode %q", s) +} + +// Type of the value +func (m *CutoffMode) Type() string { + return "string" +} diff --git a/fs/cutoffmode_test.go b/fs/cutoffmode_test.go new file mode 100644 index 000000000..0fb7cc562 --- /dev/null +++ b/fs/cutoffmode_test.go @@ -0,0 +1,6 @@ +package fs + +import "github.com/spf13/pflag" + +// Check it satisfies the interface +var _ pflag.Value = (*CutoffMode)(nil) diff --git a/fs/maxtransfermode.go b/fs/maxtransfermode.go deleted file mode 100644 index 7d38e07b5..000000000 --- a/fs/maxtransfermode.go +++ /dev/null @@ -1,12 +0,0 @@ -package fs - -// MaxTransferMode describes the possible delete modes in the config -type MaxTransferMode byte - -// MaxTransferMode constants -const ( - MaxTransferModeHard MaxTransferMode = iota - MaxTransferModeSoft - MaxTransferModeCautious - MaxTransferModeDefault = MaxTransferModeHard -) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index e4702fcc9..ce4f24a5c 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -362,12 +362,11 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj // Try server side copy first - if has optional interface and // is same underlying remote actionTaken = "Copied (server side copy)" + if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) || + (fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) { + return nil, accounting.ErrorMaxTransferLimitReached + } if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) { - // Check transfer limit for server side copies - if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) || - (fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) { - return nil, accounting.ErrorMaxTransferLimitReached - } in := tr.Account(nil) // account the transfer in.ServerSideCopyStart() newDst, err = doCopy(ctx, src, remote) @@ -386,10 +385,6 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj } // If can't server side copy, do it manually if err == fs.ErrorCantCopy { - if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) || - (fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) { - return nil, accounting.ErrorMaxTransferLimitReached - } if doMultiThreadCopy(f, src) { // Number of streams proportional to size streams := src.Size() / int64(fs.Config.MultiThreadCutoff) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 539fdc564..192dd6610 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1533,11 +1533,11 @@ func TestCopyFileMaxTransfer(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() old := fs.Config.MaxTransfer - oldMode := fs.Config.MaxTransferMode + oldMode := fs.Config.CutoffMode defer func() { fs.Config.MaxTransfer = old - fs.Config.MaxTransferMode = oldMode + fs.Config.CutoffMode = oldMode accounting.Stats(context.Background()).ResetCounters() }() @@ -1550,7 +1550,7 @@ func TestCopyFileMaxTransfer(t *testing.T) { rfile2.Path = "sub/file2" fs.Config.MaxTransfer = 15 - fs.Config.MaxTransferMode = fs.MaxTransferModeHard + fs.Config.CutoffMode = fs.CutoffModeHard accounting.Stats(context.Background()).ResetCounters() err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile1.Path, file1.Path) @@ -1563,19 +1563,19 @@ func TestCopyFileMaxTransfer(t *testing.T) { err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path) fstest.CheckItems(t, r.Flocal, file1, file2) fstest.CheckItems(t, r.Fremote, rfile1) - assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err) + assert.Contains(t, err.Error(), "Max transfer limit reached") assert.True(t, fserrors.IsFatalError(err)) - fs.Config.MaxTransferMode = fs.MaxTransferModeCautious + fs.Config.CutoffMode = fs.CutoffModeCautious accounting.Stats(context.Background()).ResetCounters() err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path) fstest.CheckItems(t, r.Flocal, file1, file2) fstest.CheckItems(t, r.Fremote, rfile1) - assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err) + assert.Contains(t, err.Error(), "Max transfer limit reached") assert.True(t, fserrors.IsFatalError(err)) - fs.Config.MaxTransferMode = fs.MaxTransferModeSoft + fs.Config.CutoffMode = fs.CutoffModeSoft accounting.Stats(context.Background()).ResetCounters() err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)