yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fs: Use --cutoff-mode hard,soft,catious instead of 3 --max-transfer-mode flags 

Fixes #2672
s3-about
Franklyn Tackitt 2020-02-25 10:03:21 -07:00 committed by Nick Craig-Wood
parent 6f1766dd9e
commit 2b3d13a841
10 changed files with 86 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/content/docs.md
View File

@ -736,18 +736,18 @@ When the limit is reached all transfers will stop immediately.
Rclone will exit with exit code 8 if the transfer limit is reached.
### --max-transfer-(hard,soft,cautious) ###
### --cutoff-mode=hard|soft|cautious ###
This modifies the behavior of `--max-transfer`
Defaults to `--max-transfer-hard`.
Defaults to `--cutoff-mode=hard`.
Specifiying `--max-transfer-hard` will stop transferring immediately
Specifiying `--cutoff-mode=hard` will stop transferring immediately
when Rclone reaches the limit.
Specifiying `--max-transfer-soft` will stop starting new transfers
Specifiying `--cutoff-mode=soft` will stop starting new transfers
when Rclone reaches the limit.
Specifiying `--max-transfer-cautious` will try to prevent Rclone
Specifiying `--cutoff-mode=cautious` will try to prevent Rclone
from reaching the limit.
### --modify-window=TIME ###

2
fs/accounting/accounting.go
View File

@ -63,7 +63,7 @@ func newAccountSizeName(stats *StatsInfo, in io.ReadCloser, size int64, name str
lpTime: time.Now(),
max: -1,
}
if fs.Config.MaxTransferMode != fs.MaxTransferModeSoft {
if fs.Config.CutoffMode == fs.CutoffModeHard {
acc.max = int64((fs.Config.MaxTransfer))
}
go acc.averageLoop()

6
fs/accounting/accounting_test.go
View File

@ -197,12 +197,12 @@ func TestAccountAccounter(t *testing.T) {
func TestAccountMaxTransfer(t *testing.T) {
old := fs.Config.MaxTransfer
oldMode := fs.Config.MaxTransferMode
oldMode := fs.Config.CutoffMode
fs.Config.MaxTransfer = 15
defer func() {
fs.Config.MaxTransfer = old
fs.Config.MaxTransferMode = oldMode
fs.Config.CutoffMode = oldMode
}()
in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
@ -222,7 +222,7 @@ func TestAccountMaxTransfer(t *testing.T) {
assert.Equal(t, ErrorMaxTransferLimitReached, err)
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
fs.Config.CutoffMode = fs.CutoffModeSoft
stats = NewStats()
acc = newAccountSizeName(stats, in, 1, "test")

2
fs/config.go
View File

@ -93,7 +93,7 @@ type ConfigInfo struct {
UseServerModTime bool
MaxTransfer SizeSuffix
MaxDuration time.Duration
MaxTransferMode MaxTransferMode
CutoffMode CutoffMode
MaxBacklog int
MaxStatsGroups int
StatsOneLine bool

21
fs/config/configflags/configflags.go
View File

@ -29,9 +29,6 @@ var (
deleteAfter bool
bindAddr string
disableFeatures string
maxTransferHard bool
maxTransferSoft bool
maxTransferCautious bool
)
// AddFlags adds the non filing system specific flags to the command
@ -97,9 +94,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
flags.BoolVarP(flagSet, &maxTransferHard, "max-transfer-hard", "", false, "When transferring, stop immediately when --max-transfer is reached")
flags.BoolVarP(flagSet, &maxTransferSoft, "max-transfer-soft", "", false, "When transferring, stop starting new transfers when --max-transfer is reached")
flags.BoolVarP(flagSet, &maxTransferCautious, "max-transfer-cautious", "", false, "When transferring, try to avoid reaching --max-transfer")
flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
@ -184,20 +179,6 @@ func SetFlags() {
fs.Config.DeleteMode = fs.DeleteModeDefault
}
switch {
case maxTransferHard && (maxTransferSoft || maxTransferCautious),
maxTransferSoft && maxTransferCautious:
log.Fatalf(`Only one of --max-trans fer-hard, --max-transfer-soft or --max-transfer-cautious can be used.`)
case maxTransferHard:
fs.Config.MaxTransferMode = fs.MaxTransferModeHard
case maxTransferSoft:
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
case maxTransferCautious:
fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
default:
fs.Config.MaxTransferMode = fs.MaxTransferModeDefault
}
if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}

49
fs/cutoffmode.go Normal file
View File

@ -0,0 +1,49 @@
package fs
import (
"fmt"
"strings"
"github.com/pkg/errors"
)
// CutoffMode describes the possible delete modes in the config
type CutoffMode byte
// MaxTransferMode constants
const (
CutoffModeHard CutoffMode = iota
CutoffModeSoft
CutoffModeCautious
CutoffModeDefault = CutoffModeHard
)
var cutoffModeToString = []string{
CutoffModeHard: "HARD",
CutoffModeSoft: "SOFT",
CutoffModeCautious: "CAUTIOUS",
}
// String turns a LogLevel into a string
func (m CutoffMode) String() string {
if m >= CutoffMode(len(cutoffModeToString)) {
return fmt.Sprintf("CutoffMode(%d)", m)
}
return cutoffModeToString[m]
}
// Set a LogLevel
func (m *CutoffMode) Set(s string) error {
for n, name := range cutoffModeToString {
if s != "" && name == strings.ToUpper(s) {
*m = CutoffMode(n)
return nil
}
}
return errors.Errorf("Unknown cutoff mode %q", s)
}
// Type of the value
func (m *CutoffMode) Type() string {
return "string"
}

6
fs/cutoffmode_test.go Normal file
View File

@ -0,0 +1,6 @@
package fs
import "github.com/spf13/pflag"
// Check it satisfies the interface
var _ pflag.Value = (*CutoffMode)(nil)

12
fs/maxtransfermode.go
View File

@ -1,12 +0,0 @@
package fs
// MaxTransferMode describes the possible delete modes in the config
type MaxTransferMode byte
// MaxTransferMode constants
const (
MaxTransferModeHard MaxTransferMode = iota
MaxTransferModeSoft
MaxTransferModeCautious
MaxTransferModeDefault = MaxTransferModeHard
)

9
fs/operations/operations.go
View File

@ -362,12 +362,11 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
// Try server side copy first - if has optional interface and
// is same underlying remote
actionTaken = "Copied (server side copy)"
if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
// Check transfer limit for server side copies
if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
(fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
return nil, accounting.ErrorMaxTransferLimitReached
}
if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
in := tr.Account(nil) // account the transfer
in.ServerSideCopyStart()
newDst, err = doCopy(ctx, src, remote)
@ -386,10 +385,6 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// If can't server side copy, do it manually
if err == fs.ErrorCantCopy {
if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
return nil, accounting.ErrorMaxTransferLimitReached
}
if doMultiThreadCopy(f, src) {
// Number of streams proportional to size
streams := src.Size() / int64(fs.Config.MultiThreadCutoff)

14
fs/operations/operations_test.go
View File

@ -1533,11 +1533,11 @@ func TestCopyFileMaxTransfer(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
old := fs.Config.MaxTransfer
oldMode := fs.Config.MaxTransferMode
oldMode := fs.Config.CutoffMode
defer func() {
fs.Config.MaxTransfer = old
fs.Config.MaxTransferMode = oldMode
fs.Config.CutoffMode = oldMode
accounting.Stats(context.Background()).ResetCounters()
}()
@ -1550,7 +1550,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
rfile2.Path = "sub/file2"
fs.Config.MaxTransfer = 15
fs.Config.MaxTransferMode = fs.MaxTransferModeHard
fs.Config.CutoffMode = fs.CutoffModeHard
accounting.Stats(context.Background()).ResetCounters()
err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile1.Path, file1.Path)
@ -1563,19 +1563,19 @@ func TestCopyFileMaxTransfer(t *testing.T) {
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, rfile1)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
assert.Contains(t, err.Error(), "Max transfer limit reached")
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
fs.Config.CutoffMode = fs.CutoffModeCautious
accounting.Stats(context.Background()).ResetCounters()
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, rfile1)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
assert.Contains(t, err.Error(), "Max transfer limit reached")
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
fs.Config.CutoffMode = fs.CutoffModeSoft
accounting.Stats(context.Background()).ResetCounters()
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)