yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fs: Use --cutoff-mode hard,soft,catious instead of 3 --max-transfer-mode flags 

Fixes #2672
s3-about
Franklyn Tackitt 2020-02-25 10:03:21 -07:00 committed by Nick Craig-Wood
parent 6f1766dd9e
commit 2b3d13a841
10 changed files with 86 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/content/docs.md
View File

@ -736,18 +736,18 @@ When the limit is reached all transfers will stop immediately.
Rclone will exit with exit code 8 if the transfer limit is reached.
### --max-transfer-(hard,soft,cautious) ###
### --cutoff-mode=hard|soft|cautious ###
This modifies the behavior of `--max-transfer`
Defaults to `--max-transfer-hard`.
Defaults to `--cutoff-mode=hard`.
Specifiying `--max-transfer-hard` will stop transferring immediately
Specifiying `--cutoff-mode=hard` will stop transferring immediately
when Rclone reaches the limit.
Specifiying `--max-transfer-soft` will stop starting new transfers
Specifiying `--cutoff-mode=soft` will stop starting new transfers
when Rclone reaches the limit.
Specifiying `--max-transfer-cautious` will try to prevent Rclone
Specifiying `--cutoff-mode=cautious` will try to prevent Rclone
from reaching the limit.
### --modify-window=TIME ###

2
fs/accounting/accounting.go
View File

@ -63,7 +63,7 @@ func newAccountSizeName(stats *StatsInfo, in io.ReadCloser, size int64, name str
lpTime: time.Now(),
max: -1,
}
if fs.Config.MaxTransferMode != fs.MaxTransferModeSoft {
if fs.Config.CutoffMode == fs.CutoffModeHard {
acc.max = int64((fs.Config.MaxTransfer))
}
go acc.averageLoop()

6
fs/accounting/accounting_test.go
View File

@ -197,12 +197,12 @@ func TestAccountAccounter(t *testing.T) {
func TestAccountMaxTransfer(t *testing.T) {
old := fs.Config.MaxTransfer
oldMode := fs.Config.MaxTransferMode
oldMode := fs.Config.CutoffMode
fs.Config.MaxTransfer = 15
defer func() {
fs.Config.MaxTransfer = old
fs.Config.MaxTransferMode = oldMode
fs.Config.CutoffMode = oldMode
}()
in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
@ -222,7 +222,7 @@ func TestAccountMaxTransfer(t *testing.T) {
assert.Equal(t, ErrorMaxTransferLimitReached, err)
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
fs.Config.CutoffMode = fs.CutoffModeSoft
stats = NewStats()
acc = newAccountSizeName(stats, in, 1, "test")

2
fs/config.go
View File

@ -93,7 +93,7 @@ type ConfigInfo struct {
UseServerModTime bool
MaxTransfer SizeSuffix
MaxDuration time.Duration
MaxTransferMode MaxTransferMode
CutoffMode CutoffMode
MaxBacklog int
MaxStatsGroups int
StatsOneLine bool

39
fs/config/configflags/configflags.go
View File

@ -20,18 +20,15 @@ import (
var (
// these will get interpreted into fs.Config via SetFlags() below
verbose int
quiet bool
dumpHeaders bool
dumpBodies bool
deleteBefore bool
deleteDuring bool
deleteAfter bool
bindAddr string
disableFeatures string
maxTransferHard bool
maxTransferSoft bool
maxTransferCautious bool
verbose int
quiet bool
dumpHeaders bool
dumpBodies bool
deleteBefore bool
deleteDuring bool
deleteAfter bool
bindAddr string
disableFeatures string
)
// AddFlags adds the non filing system specific flags to the command
@ -97,9 +94,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
flags.BoolVarP(flagSet, &maxTransferHard, "max-transfer-hard", "", false, "When transferring, stop immediately when --max-transfer is reached")
flags.BoolVarP(flagSet, &maxTransferSoft, "max-transfer-soft", "", false, "When transferring, stop starting new transfers when --max-transfer is reached")
flags.BoolVarP(flagSet, &maxTransferCautious, "max-transfer-cautious", "", false, "When transferring, try to avoid reaching --max-transfer")
flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
@ -184,20 +179,6 @@ func SetFlags() {
fs.Config.DeleteMode = fs.DeleteModeDefault
}
switch {
case maxTransferHard && (maxTransferSoft || maxTransferCautious),
maxTransferSoft && maxTransferCautious:
log.Fatalf(`Only one of --max-trans fer-hard, --max-transfer-soft or --max-transfer-cautious can be used.`)
case maxTransferHard:
fs.Config.MaxTransferMode = fs.MaxTransferModeHard
case maxTransferSoft:
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
case maxTransferCautious:
fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
default:
fs.Config.MaxTransferMode = fs.MaxTransferModeDefault
}
if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}

49
fs/cutoffmode.go Normal file
View File

@ -0,0 +1,49 @@
package fs
import (
"fmt"
"strings"
"github.com/pkg/errors"
)
// CutoffMode describes the possible delete modes in the config
type CutoffMode byte
// MaxTransferMode constants
const (
CutoffModeHard CutoffMode = iota
CutoffModeSoft
CutoffModeCautious
CutoffModeDefault = CutoffModeHard
)
var cutoffModeToString = []string{
CutoffModeHard: "HARD",
CutoffModeSoft: "SOFT",
CutoffModeCautious: "CAUTIOUS",
}
// String turns a LogLevel into a string
func (m CutoffMode) String() string {
if m >= CutoffMode(len(cutoffModeToString)) {
return fmt.Sprintf("CutoffMode(%d)", m)
}
return cutoffModeToString[m]
}
// Set a LogLevel
func (m *CutoffMode) Set(s string) error {
for n, name := range cutoffModeToString {
if s != "" && name == strings.ToUpper(s) {
*m = CutoffMode(n)
return nil
}
}
return errors.Errorf("Unknown cutoff mode %q", s)
}
// Type of the value
func (m *CutoffMode) Type() string {
return "string"
}

6
fs/cutoffmode_test.go Normal file
View File

@ -0,0 +1,6 @@
package fs
import "github.com/spf13/pflag"
// Check it satisfies the interface
var _ pflag.Value = (*CutoffMode)(nil)

12
fs/maxtransfermode.go
View File

@ -1,12 +0,0 @@
package fs
// MaxTransferMode describes the possible delete modes in the config
type MaxTransferMode byte
// MaxTransferMode constants
const (
MaxTransferModeHard MaxTransferMode = iota
MaxTransferModeSoft
MaxTransferModeCautious
MaxTransferModeDefault = MaxTransferModeHard
)

13
fs/operations/operations.go
View File

@ -362,12 +362,11 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
// Try server side copy first - if has optional interface and
// is same underlying remote
actionTaken = "Copied (server side copy)"
if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
(fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
return nil, accounting.ErrorMaxTransferLimitReached
}
if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
// Check transfer limit for server side copies
if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
return nil, accounting.ErrorMaxTransferLimitReached
}
in := tr.Account(nil) // account the transfer
in.ServerSideCopyStart()
newDst, err = doCopy(ctx, src, remote)
@ -386,10 +385,6 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// If can't server side copy, do it manually
if err == fs.ErrorCantCopy {
if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
(fs.Config.MaxTransferMode == fs.MaxTransferModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
return nil, accounting.ErrorMaxTransferLimitReached
}
if doMultiThreadCopy(f, src) {
// Number of streams proportional to size
streams := src.Size() / int64(fs.Config.MultiThreadCutoff)

14
fs/operations/operations_test.go
View File

@ -1533,11 +1533,11 @@ func TestCopyFileMaxTransfer(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
old := fs.Config.MaxTransfer
oldMode := fs.Config.MaxTransferMode
oldMode := fs.Config.CutoffMode
defer func() {
fs.Config.MaxTransfer = old
fs.Config.MaxTransferMode = oldMode
fs.Config.CutoffMode = oldMode
accounting.Stats(context.Background()).ResetCounters()
}()
@ -1550,7 +1550,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
rfile2.Path = "sub/file2"
fs.Config.MaxTransfer = 15
fs.Config.MaxTransferMode = fs.MaxTransferModeHard
fs.Config.CutoffMode = fs.CutoffModeHard
accounting.Stats(context.Background()).ResetCounters()
err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile1.Path, file1.Path)
@ -1563,19 +1563,19 @@ func TestCopyFileMaxTransfer(t *testing.T) {
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, rfile1)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
assert.Contains(t, err.Error(), "Max transfer limit reached")
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeCautious
fs.Config.CutoffMode = fs.CutoffModeCautious
accounting.Stats(context.Background()).ResetCounters()
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, rfile1)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
assert.Contains(t, err.Error(), "Max transfer limit reached")
assert.True(t, fserrors.IsFatalError(err))
fs.Config.MaxTransferMode = fs.MaxTransferModeSoft
fs.Config.CutoffMode = fs.CutoffModeSoft
accounting.Stats(context.Background()).ResetCounters()
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)