Allow all options to be set from environment variables
The option names are munged changing - to _ making upper case and prepending RCLONE_. The values are as parsed by pflag.s3-about
Nick Craig-Wood
parent
3b0f944e23
commit
0d75d2585f
|
|
@ -31,7 +31,6 @@ import (
|
|||
"github.com/ncw/rclone/pacer"
|
||||
"github.com/ncw/rclone/rest"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
|
|
@ -51,7 +50,7 @@ const (
|
|||
var (
|
||||
// Flags
|
||||
tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
|
||||
uploadWaitPerGB = pflag.DurationP("acd-upload-wait-per-gb", "", 180*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
|
||||
uploadWaitPerGB = fs.DurationP("acd-upload-wait-per-gb", "", 180*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
|
||||
// Description of how to auth for this app
|
||||
acdConfig = &oauth2.Config{
|
||||
Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
|
||||
|
|
@ -85,7 +84,7 @@ func init() {
|
|||
Help: "Amazon Application Client Secret - leave blank normally.",
|
||||
}},
|
||||
})
|
||||
pflag.VarP(&tempLinkThreshold, "acd-templink-threshold", "", "Files >= this size will be downloaded via their tempLink.")
|
||||
fs.VarP(&tempLinkThreshold, "acd-templink-threshold", "", "Files >= this size will be downloaded via their tempLink.")
|
||||
}
|
||||
|
||||
// Fs represents a remote acd server
|
||||
|
|
|
|||
9
b2/b2.go
9
b2/b2.go
|
|
@ -25,7 +25,6 @@ import (
|
|||
"github.com/ncw/rclone/pacer"
|
||||
"github.com/ncw/rclone/rest"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -50,8 +49,8 @@ var (
|
|||
minChunkSize = fs.SizeSuffix(100E6)
|
||||
chunkSize = fs.SizeSuffix(96 * 1024 * 1024)
|
||||
uploadCutoff = fs.SizeSuffix(200E6)
|
||||
b2TestMode = pflag.StringP("b2-test-mode", "", "", "A flag string for X-Bz-Test-Mode header.")
|
||||
b2Versions = pflag.BoolP("b2-versions", "", false, "Include old versions in directory listings.")
|
||||
b2TestMode = fs.StringP("b2-test-mode", "", "", "A flag string for X-Bz-Test-Mode header.")
|
||||
b2Versions = fs.BoolP("b2-versions", "", false, "Include old versions in directory listings.")
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
)
|
||||
|
||||
|
|
@ -73,8 +72,8 @@ func init() {
|
|||
},
|
||||
},
|
||||
})
|
||||
pflag.VarP(&uploadCutoff, "b2-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
pflag.VarP(&chunkSize, "b2-chunk-size", "", "Upload chunk size. Must fit in memory.")
|
||||
fs.VarP(&uploadCutoff, "b2-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
fs.VarP(&chunkSize, "b2-chunk-size", "", "Upload chunk size. Must fit in memory.")
|
||||
}
|
||||
|
||||
// Fs represents a remote b2 server
|
||||
|
|
|
|||
12
cmd/cmd.go
12
cmd/cmd.go
|
|
@ -26,13 +26,13 @@ import (
|
|||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
cpuProfile = pflag.StringP("cpuprofile", "", "", "Write cpu profile to file")
|
||||
memProfile = pflag.String("memprofile", "", "Write memory profile to file")
|
||||
statsInterval = pflag.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
|
||||
dataRateUnit = pflag.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
|
||||
cpuProfile = fs.StringP("cpuprofile", "", "", "Write cpu profile to file")
|
||||
memProfile = fs.StringP("memprofile", "", "", "Write memory profile to file")
|
||||
statsInterval = fs.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
|
||||
dataRateUnit = fs.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
|
||||
version bool
|
||||
logFile = pflag.StringP("log-file", "", "", "Log everything to this file")
|
||||
retries = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
|
||||
logFile = fs.StringP("log-file", "", "", "Log everything to this file")
|
||||
retries = fs.IntP("retries", "", 3, "Retry operations this many times if they fail")
|
||||
)
|
||||
|
||||
// Root is the main rclone command
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"github.com/ncw/rclone/oauthutil"
|
||||
"github.com/ncw/rclone/pacer"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/drive/v2"
|
||||
|
|
@ -42,10 +41,10 @@ const (
|
|||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
driveFullList = pflag.BoolP("drive-full-list", "", false, "Use a full listing for directory list. More data but usually quicker. (obsolete)")
|
||||
driveAuthOwnerOnly = pflag.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user. Requires drive-full-list.")
|
||||
driveUseTrash = pflag.BoolP("drive-use-trash", "", false, "Send files to the trash instead of deleting permanently.")
|
||||
driveExtensions = pflag.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
|
||||
driveFullList = fs.BoolP("drive-full-list", "", false, "Use a full listing for directory list. More data but usually quicker. (obsolete)")
|
||||
driveAuthOwnerOnly = fs.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user. Requires drive-full-list.")
|
||||
driveUseTrash = fs.BoolP("drive-use-trash", "", false, "Send files to the trash instead of deleting permanently.")
|
||||
driveExtensions = fs.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
|
||||
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
chunkSize = fs.SizeSuffix(8 * 1024 * 1024)
|
||||
|
|
@ -103,8 +102,8 @@ func init() {
|
|||
Help: "Google Application Client Secret - leave blank normally.",
|
||||
}},
|
||||
})
|
||||
pflag.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
pflag.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
|
||||
fs.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
fs.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
|
||||
|
||||
// Invert mimeTypeToExtension
|
||||
extensionToMimeType = make(map[string]string, len(mimeTypeToExtension))
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/oauthutil"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/stacktic/dropbox"
|
||||
)
|
||||
|
||||
|
|
@ -58,7 +57,7 @@ func init() {
|
|||
Help: "Dropbox App Secret - leave blank normally.",
|
||||
}},
|
||||
})
|
||||
pflag.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
|
||||
fs.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
|
||||
}
|
||||
|
||||
// Configuration helper - called after the user has put in the defaults
|
||||
|
|
|
|||
281
fs/config.go
281
fs/config.go
|
|
@ -14,7 +14,6 @@ import (
|
|||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
|
|
@ -26,7 +25,6 @@ import (
|
|||
|
||||
"github.com/Unknwon/goconfig"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/crypto/nacl/secretbox"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
|
@ -47,18 +45,6 @@ const (
|
|||
ConfigAutomatic = "config_automatic"
|
||||
)
|
||||
|
||||
// SizeSuffix is parsed by flag with k/M/G suffixes
|
||||
type SizeSuffix int64
|
||||
|
||||
// BwTimeSlot represents a bandwidth configuration at a point in time.
|
||||
type BwTimeSlot struct {
|
||||
hhmm int
|
||||
bandwidth SizeSuffix
|
||||
}
|
||||
|
||||
// BwTimetable contains all configured time slots.
|
||||
type BwTimetable []BwTimeSlot
|
||||
|
||||
// Global
|
||||
var (
|
||||
// ConfigFile is the config file data structure
|
||||
|
|
@ -70,35 +56,35 @@ var (
|
|||
// Config is the global config
|
||||
Config = &ConfigInfo{}
|
||||
// Flags
|
||||
verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
|
||||
quiet = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
|
||||
modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
|
||||
checkers = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
|
||||
transfers = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
|
||||
configFile = pflag.StringP("config", "", ConfigPath, "Config file.")
|
||||
checkSum = pflag.BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
|
||||
sizeOnly = pflag.BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
|
||||
ignoreTimes = pflag.BoolP("ignore-times", "I", false, "Don't skip files that match size and time - transfer all files")
|
||||
ignoreExisting = pflag.BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
|
||||
dryRun = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
|
||||
connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
|
||||
timeout = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
|
||||
dumpHeaders = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
|
||||
dumpBodies = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
|
||||
dumpAuth = pflag.BoolP("dump-auth", "", false, "Dump HTTP headers with auth info")
|
||||
skipVerify = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
|
||||
AskPassword = pflag.BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
|
||||
deleteBefore = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
|
||||
deleteDuring = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
|
||||
deleteAfter = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
|
||||
trackRenames = pflag.BoolP("track-renames", "", false, "When synchronizing, track file renames and do a server side move if possible")
|
||||
lowLevelRetries = pflag.IntP("low-level-retries", "", 10, "Number of low level retries to do.")
|
||||
updateOlder = pflag.BoolP("update", "u", false, "Skip files that are newer on the destination.")
|
||||
noGzip = pflag.BoolP("no-gzip-encoding", "", false, "Don't set Accept-Encoding: gzip.")
|
||||
maxDepth = pflag.IntP("max-depth", "", -1, "If set limits the recursion depth to this.")
|
||||
ignoreSize = pflag.BoolP("ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
|
||||
noTraverse = pflag.BoolP("no-traverse", "", false, "Don't traverse destination file system on copy.")
|
||||
noUpdateModTime = pflag.BoolP("no-update-modtime", "", false, "Don't update destination mod-time if files identical.")
|
||||
verbose = BoolP("verbose", "v", false, "Print lots more stuff")
|
||||
quiet = BoolP("quiet", "q", false, "Print as little stuff as possible")
|
||||
modifyWindow = DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
|
||||
checkers = IntP("checkers", "", 8, "Number of checkers to run in parallel.")
|
||||
transfers = IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
|
||||
configFile = StringP("config", "", ConfigPath, "Config file.")
|
||||
checkSum = BoolP("checksum", "c", false, "Skip based on checksum & size, not mod-time & size")
|
||||
sizeOnly = BoolP("size-only", "", false, "Skip based on size only, not mod-time or checksum")
|
||||
ignoreTimes = BoolP("ignore-times", "I", false, "Don't skip files that match size and time - transfer all files")
|
||||
ignoreExisting = BoolP("ignore-existing", "", false, "Skip all files that exist on destination")
|
||||
dryRun = BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
|
||||
connectTimeout = DurationP("contimeout", "", 60*time.Second, "Connect timeout")
|
||||
timeout = DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
|
||||
dumpHeaders = BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
|
||||
dumpBodies = BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
|
||||
dumpAuth = BoolP("dump-auth", "", false, "Dump HTTP headers with auth info")
|
||||
skipVerify = BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
|
||||
AskPassword = BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
|
||||
deleteBefore = BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
|
||||
deleteDuring = BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
|
||||
deleteAfter = BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
|
||||
trackRenames = BoolP("track-renames", "", false, "When synchronizing, track file renames and do a server side move if possible")
|
||||
lowLevelRetries = IntP("low-level-retries", "", 10, "Number of low level retries to do.")
|
||||
updateOlder = BoolP("update", "u", false, "Skip files that are newer on the destination.")
|
||||
noGzip = BoolP("no-gzip-encoding", "", false, "Don't set Accept-Encoding: gzip.")
|
||||
maxDepth = IntP("max-depth", "", -1, "If set limits the recursion depth to this.")
|
||||
ignoreSize = BoolP("ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
|
||||
noTraverse = BoolP("no-traverse", "", false, "Don't traverse destination file system on copy.")
|
||||
noUpdateModTime = BoolP("no-update-modtime", "", false, "Don't update destination mod-time if files identical.")
|
||||
bwLimit BwTimetable
|
||||
|
||||
// Key to use for password en/decryption.
|
||||
|
|
@ -107,216 +93,9 @@ var (
|
|||
)
|
||||
|
||||
func init() {
|
||||
pflag.VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
|
||||
VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
|
||||
}
|
||||
|
||||
// Turn SizeSuffix into a string and a suffix
|
||||
func (x SizeSuffix) string() (string, string) {
|
||||
scaled := float64(0)
|
||||
suffix := ""
|
||||
switch {
|
||||
case x < 0:
|
||||
return "off", ""
|
||||
case x == 0:
|
||||
return "0", ""
|
||||
case x < 1024:
|
||||
scaled = float64(x)
|
||||
suffix = ""
|
||||
case x < 1024*1024:
|
||||
scaled = float64(x) / 1024
|
||||
suffix = "k"
|
||||
case x < 1024*1024*1024:
|
||||
scaled = float64(x) / 1024 / 1024
|
||||
suffix = "M"
|
||||
default:
|
||||
scaled = float64(x) / 1024 / 1024 / 1024
|
||||
suffix = "G"
|
||||
}
|
||||
if math.Floor(scaled) == scaled {
|
||||
return fmt.Sprintf("%.0f", scaled), suffix
|
||||
}
|
||||
return fmt.Sprintf("%.3f", scaled), suffix
|
||||
}
|
||||
|
||||
// String turns SizeSuffix into a string
|
||||
func (x SizeSuffix) String() string {
|
||||
val, suffix := x.string()
|
||||
return val + suffix
|
||||
}
|
||||
|
||||
// Unit turns SizeSuffix into a string with a unit
|
||||
func (x SizeSuffix) Unit(unit string) string {
|
||||
val, suffix := x.string()
|
||||
if val == "off" {
|
||||
return val
|
||||
}
|
||||
return val + " " + suffix + unit
|
||||
}
|
||||
|
||||
// Set a SizeSuffix
|
||||
func (x *SizeSuffix) Set(s string) error {
|
||||
if len(s) == 0 {
|
||||
return errors.New("empty string")
|
||||
}
|
||||
if strings.ToLower(s) == "off" {
|
||||
*x = -1
|
||||
return nil
|
||||
}
|
||||
suffix := s[len(s)-1]
|
||||
suffixLen := 1
|
||||
var multiplier float64
|
||||
switch suffix {
|
||||
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
|
||||
suffixLen = 0
|
||||
multiplier = 1 << 10
|
||||
case 'b', 'B':
|
||||
multiplier = 1
|
||||
case 'k', 'K':
|
||||
multiplier = 1 << 10
|
||||
case 'm', 'M':
|
||||
multiplier = 1 << 20
|
||||
case 'g', 'G':
|
||||
multiplier = 1 << 30
|
||||
default:
|
||||
return errors.Errorf("bad suffix %q", suffix)
|
||||
}
|
||||
s = s[:len(s)-suffixLen]
|
||||
value, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value < 0 {
|
||||
return errors.Errorf("size can't be negative %q", s)
|
||||
}
|
||||
value *= multiplier
|
||||
*x = SizeSuffix(value)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Type of the value
|
||||
func (x *SizeSuffix) Type() string {
|
||||
return "int64"
|
||||
}
|
||||
|
||||
// Check it satisfies the interface
|
||||
var _ pflag.Value = (*SizeSuffix)(nil)
|
||||
|
||||
// String returns a printable representation of BwTimetable.
|
||||
func (x BwTimetable) String() string {
|
||||
ret := []string{}
|
||||
for _, ts := range x {
|
||||
ret = append(ret, fmt.Sprintf("%04.4d,%s", ts.hhmm, ts.bandwidth.String()))
|
||||
}
|
||||
return strings.Join(ret, " ")
|
||||
}
|
||||
|
||||
// Set the bandwidth timetable.
|
||||
func (x *BwTimetable) Set(s string) error {
|
||||
// The timetable is formatted as:
|
||||
// "hh:mm,bandwidth hh:mm,banwidth..." ex: "10:00,10G 11:30,1G 18:00,off"
|
||||
// If only a single bandwidth identifier is provided, we assume constant bandwidth.
|
||||
|
||||
if len(s) == 0 {
|
||||
return errors.New("empty string")
|
||||
}
|
||||
// Single value without time specification.
|
||||
if !strings.Contains(s, " ") && !strings.Contains(s, ",") {
|
||||
ts := BwTimeSlot{}
|
||||
if err := ts.bandwidth.Set(s); err != nil {
|
||||
return err
|
||||
}
|
||||
ts.hhmm = 0
|
||||
*x = BwTimetable{ts}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, tok := range strings.Split(s, " ") {
|
||||
tv := strings.Split(tok, ",")
|
||||
|
||||
// Format must be HH:MM,BW
|
||||
if len(tv) != 2 {
|
||||
return errors.Errorf("invalid time/bandwidth specification: %q", tok)
|
||||
}
|
||||
|
||||
// Basic timespec sanity checking
|
||||
hhmm := tv[0]
|
||||
if len(hhmm) != 5 {
|
||||
return errors.Errorf("invalid time specification (hh:mm): %q", hhmm)
|
||||
}
|
||||
hh, err := strconv.Atoi(hhmm[0:2])
|
||||
if err != nil {
|
||||
return errors.Errorf("invalid hour in time specification %q: %v", hhmm, err)
|
||||
}
|
||||
if hh < 0 || hh > 23 {
|
||||
return errors.Errorf("invalid hour (must be between 00 and 23): %q", hh)
|
||||
}
|
||||
mm, err := strconv.Atoi(hhmm[3:])
|
||||
if err != nil {
|
||||
return errors.Errorf("invalid minute in time specification: %q: %v", hhmm, err)
|
||||
}
|
||||
if mm < 0 || mm > 59 {
|
||||
return errors.Errorf("invalid minute (must be between 00 and 59): %q", hh)
|
||||
}
|
||||
|
||||
ts := BwTimeSlot{
|
||||
hhmm: (hh * 100) + mm,
|
||||
}
|
||||
// Bandwidth limit for this time slot.
|
||||
if err := ts.bandwidth.Set(tv[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
*x = append(*x, ts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LimitAt returns a BwTimeSlot for the time requested.
|
||||
func (x BwTimetable) LimitAt(tt time.Time) BwTimeSlot {
|
||||
// If the timetable is empty, we return an unlimited BwTimeSlot starting at midnight.
|
||||
if len(x) == 0 {
|
||||
return BwTimeSlot{hhmm: 0, bandwidth: -1}
|
||||
}
|
||||
|
||||
hhmm := tt.Hour()*100 + tt.Minute()
|
||||
|
||||
// By default, we return the last element in the timetable. This
|
||||
// satisfies two conditions: 1) If there's only one element it
|
||||
// will always be selected, and 2) The last element of the table
|
||||
// will "wrap around" until overriden by an earlier time slot.
|
||||
// there's only one time slot in the timetable.
|
||||
ret := x[len(x)-1]
|
||||
|
||||
mindif := 0
|
||||
first := true
|
||||
|
||||
// Look for most recent time slot.
|
||||
for _, ts := range x {
|
||||
// Ignore the past
|
||||
if hhmm < ts.hhmm {
|
||||
continue
|
||||
}
|
||||
dif := ((hhmm / 100 * 60) + (hhmm % 100)) - ((ts.hhmm / 100 * 60) + (ts.hhmm % 100))
|
||||
if first {
|
||||
mindif = dif
|
||||
first = false
|
||||
}
|
||||
if dif <= mindif {
|
||||
mindif = dif
|
||||
ret = ts
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Type of the value
|
||||
func (x BwTimetable) Type() string {
|
||||
return "BwTimetable"
|
||||
}
|
||||
|
||||
// Check it satisfies the interface
|
||||
var _ pflag.Value = (*BwTimetable)(nil)
|
||||
|
||||
// crypt internals
|
||||
var (
|
||||
cryptKey = []byte{
|
||||
|
|
|
|||
|
|
@ -4,190 +4,11 @@ import (
|
|||
"bytes"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSizeSuffixString(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in float64
|
||||
want string
|
||||
}{
|
||||
{0, "0"},
|
||||
{102, "102"},
|
||||
{1024, "1k"},
|
||||
{1024 * 1024, "1M"},
|
||||
{1024 * 1024 * 1024, "1G"},
|
||||
{10 * 1024 * 1024 * 1024, "10G"},
|
||||
{10.1 * 1024 * 1024 * 1024, "10.100G"},
|
||||
{-1, "off"},
|
||||
{-100, "off"},
|
||||
} {
|
||||
ss := SizeSuffix(test.in)
|
||||
got := ss.String()
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeSuffixUnit(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in float64
|
||||
want string
|
||||
}{
|
||||
{0, "0 Bytes"},
|
||||
{102, "102 Bytes"},
|
||||
{1024, "1 kBytes"},
|
||||
{1024 * 1024, "1 MBytes"},
|
||||
{1024 * 1024 * 1024, "1 GBytes"},
|
||||
{10 * 1024 * 1024 * 1024, "10 GBytes"},
|
||||
{10.1 * 1024 * 1024 * 1024, "10.100 GBytes"},
|
||||
{-1, "off"},
|
||||
{-100, "off"},
|
||||
} {
|
||||
ss := SizeSuffix(test.in)
|
||||
got := ss.Unit("Bytes")
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeSuffixSet(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want int64
|
||||
err bool
|
||||
}{
|
||||
{"0", 0, false},
|
||||
{"1b", 1, false},
|
||||
{"102B", 102, false},
|
||||
{"0.1k", 102, false},
|
||||
{"0.1", 102, false},
|
||||
{"1K", 1024, false},
|
||||
{"1", 1024, false},
|
||||
{"2.5", 1024 * 2.5, false},
|
||||
{"1M", 1024 * 1024, false},
|
||||
{"1.g", 1024 * 1024 * 1024, false},
|
||||
{"10G", 10 * 1024 * 1024 * 1024, false},
|
||||
{"off", -1, false},
|
||||
{"OFF", -1, false},
|
||||
{"", 0, true},
|
||||
{"1p", 0, true},
|
||||
{"1.p", 0, true},
|
||||
{"1p", 0, true},
|
||||
{"-1K", 0, true},
|
||||
} {
|
||||
ss := SizeSuffix(0)
|
||||
err := ss.Set(test.in)
|
||||
if test.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, test.want, int64(ss))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBwTimetableSet(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want BwTimetable
|
||||
err bool
|
||||
}{
|
||||
{"", BwTimetable{}, true},
|
||||
{"0", BwTimetable{BwTimeSlot{hhmm: 0, bandwidth: 0}}, false},
|
||||
{"666", BwTimetable{BwTimeSlot{hhmm: 0, bandwidth: 666 * 1024}}, false},
|
||||
{"10:20,666", BwTimetable{BwTimeSlot{hhmm: 1020, bandwidth: 666 * 1024}}, false},
|
||||
{
|
||||
"11:00,333 13:40,666 23:50,10M 23:59,off",
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1340, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: 10 * 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2359, bandwidth: -1},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{"bad,bad", BwTimetable{}, true},
|
||||
{"bad bad", BwTimetable{}, true},
|
||||
{"bad", BwTimetable{}, true},
|
||||
{"1000X", BwTimetable{}, true},
|
||||
{"2401,666", BwTimetable{}, true},
|
||||
{"1061,666", BwTimetable{}, true},
|
||||
} {
|
||||
tt := BwTimetable{}
|
||||
err := tt.Set(test.in)
|
||||
if test.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, test.want, tt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBwTimetableLimitAt(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
tt BwTimetable
|
||||
now time.Time
|
||||
want BwTimeSlot
|
||||
}{
|
||||
{
|
||||
BwTimetable{},
|
||||
time.Date(2017, time.April, 20, 15, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 0, bandwidth: -1},
|
||||
},
|
||||
{
|
||||
BwTimetable{BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024}},
|
||||
time.Date(2017, time.April, 20, 15, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 10, 15, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 11, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 13, 1, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 23, 59, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
} {
|
||||
slot := test.tt.LimitAt(test.now)
|
||||
assert.Equal(t, test.want, slot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObscure(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
|
|
|
|||
29
fs/filter.go
29
fs/filter.go
|
|
@ -13,31 +13,30 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// Global
|
||||
var (
|
||||
// Flags
|
||||
deleteExcluded = pflag.BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync")
|
||||
filterRule = pflag.StringArrayP("filter", "f", nil, "Add a file-filtering rule")
|
||||
filterFrom = pflag.StringArrayP("filter-from", "", nil, "Read filtering patterns from a file")
|
||||
excludeRule = pflag.StringArrayP("exclude", "", nil, "Exclude files matching pattern")
|
||||
excludeFrom = pflag.StringArrayP("exclude-from", "", nil, "Read exclude patterns from file")
|
||||
includeRule = pflag.StringArrayP("include", "", nil, "Include files matching pattern")
|
||||
includeFrom = pflag.StringArrayP("include-from", "", nil, "Read include patterns from file")
|
||||
filesFrom = pflag.StringArrayP("files-from", "", nil, "Read list of source-file names from file")
|
||||
minAge = pflag.StringP("min-age", "", "", "Don't transfer any file younger than this in s or suffix ms|s|m|h|d|w|M|y")
|
||||
maxAge = pflag.StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
|
||||
deleteExcluded = BoolP("delete-excluded", "", false, "Delete files on dest excluded from sync")
|
||||
filterRule = StringArrayP("filter", "f", nil, "Add a file-filtering rule")
|
||||
filterFrom = StringArrayP("filter-from", "", nil, "Read filtering patterns from a file")
|
||||
excludeRule = StringArrayP("exclude", "", nil, "Exclude files matching pattern")
|
||||
excludeFrom = StringArrayP("exclude-from", "", nil, "Read exclude patterns from file")
|
||||
includeRule = StringArrayP("include", "", nil, "Include files matching pattern")
|
||||
includeFrom = StringArrayP("include-from", "", nil, "Read include patterns from file")
|
||||
filesFrom = StringArrayP("files-from", "", nil, "Read list of source-file names from file")
|
||||
minAge = StringP("min-age", "", "", "Don't transfer any file younger than this in s or suffix ms|s|m|h|d|w|M|y")
|
||||
maxAge = StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
|
||||
minSize = SizeSuffix(-1)
|
||||
maxSize = SizeSuffix(-1)
|
||||
dumpFilters = pflag.BoolP("dump-filters", "", false, "Dump the filters to the output")
|
||||
//cvsExclude = pflag.BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
|
||||
dumpFilters = BoolP("dump-filters", "", false, "Dump the filters to the output")
|
||||
//cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
|
||||
)
|
||||
|
||||
func init() {
|
||||
pflag.VarP(&minSize, "min-size", "", "Don't transfer any file smaller than this in k or suffix b|k|M|G")
|
||||
pflag.VarP(&maxSize, "max-size", "", "Don't transfer any file larger than this in k or suffix b|k|M|G")
|
||||
VarP(&minSize, "min-size", "", "Don't transfer any file smaller than this in k or suffix b|k|M|G")
|
||||
VarP(&maxSize, "max-size", "", "Don't transfer any file larger than this in k or suffix b|k|M|G")
|
||||
}
|
||||
|
||||
// rule is one filter rule
|
||||
|
|
|
|||
|
|
@ -0,0 +1,315 @@
|
|||
// This contains helper functions for managing flags
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// SizeSuffix is parsed by flag with k/M/G suffixes
|
||||
type SizeSuffix int64
|
||||
|
||||
// Turn SizeSuffix into a string and a suffix
|
||||
func (x SizeSuffix) string() (string, string) {
|
||||
scaled := float64(0)
|
||||
suffix := ""
|
||||
switch {
|
||||
case x < 0:
|
||||
return "off", ""
|
||||
case x == 0:
|
||||
return "0", ""
|
||||
case x < 1024:
|
||||
scaled = float64(x)
|
||||
suffix = ""
|
||||
case x < 1024*1024:
|
||||
scaled = float64(x) / 1024
|
||||
suffix = "k"
|
||||
case x < 1024*1024*1024:
|
||||
scaled = float64(x) / 1024 / 1024
|
||||
suffix = "M"
|
||||
default:
|
||||
scaled = float64(x) / 1024 / 1024 / 1024
|
||||
suffix = "G"
|
||||
}
|
||||
if math.Floor(scaled) == scaled {
|
||||
return fmt.Sprintf("%.0f", scaled), suffix
|
||||
}
|
||||
return fmt.Sprintf("%.3f", scaled), suffix
|
||||
}
|
||||
|
||||
// String turns SizeSuffix into a string
|
||||
func (x SizeSuffix) String() string {
|
||||
val, suffix := x.string()
|
||||
return val + suffix
|
||||
}
|
||||
|
||||
// Unit turns SizeSuffix into a string with a unit
|
||||
func (x SizeSuffix) Unit(unit string) string {
|
||||
val, suffix := x.string()
|
||||
if val == "off" {
|
||||
return val
|
||||
}
|
||||
return val + " " + suffix + unit
|
||||
}
|
||||
|
||||
// Set a SizeSuffix
|
||||
func (x *SizeSuffix) Set(s string) error {
|
||||
if len(s) == 0 {
|
||||
return errors.New("empty string")
|
||||
}
|
||||
if strings.ToLower(s) == "off" {
|
||||
*x = -1
|
||||
return nil
|
||||
}
|
||||
suffix := s[len(s)-1]
|
||||
suffixLen := 1
|
||||
var multiplier float64
|
||||
switch suffix {
|
||||
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
|
||||
suffixLen = 0
|
||||
multiplier = 1 << 10
|
||||
case 'b', 'B':
|
||||
multiplier = 1
|
||||
case 'k', 'K':
|
||||
multiplier = 1 << 10
|
||||
case 'm', 'M':
|
||||
multiplier = 1 << 20
|
||||
case 'g', 'G':
|
||||
multiplier = 1 << 30
|
||||
default:
|
||||
return errors.Errorf("bad suffix %q", suffix)
|
||||
}
|
||||
s = s[:len(s)-suffixLen]
|
||||
value, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value < 0 {
|
||||
return errors.Errorf("size can't be negative %q", s)
|
||||
}
|
||||
value *= multiplier
|
||||
*x = SizeSuffix(value)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Type of the value
|
||||
func (x *SizeSuffix) Type() string {
|
||||
return "int64"
|
||||
}
|
||||
|
||||
// Check it satisfies the interface
|
||||
var _ pflag.Value = (*SizeSuffix)(nil)
|
||||
|
||||
// BwTimeSlot represents a bandwidth configuration at a point in time.
|
||||
type BwTimeSlot struct {
|
||||
hhmm int
|
||||
bandwidth SizeSuffix
|
||||
}
|
||||
|
||||
// BwTimetable contains all configured time slots.
|
||||
type BwTimetable []BwTimeSlot
|
||||
|
||||
// String returns a printable representation of BwTimetable.
|
||||
func (x BwTimetable) String() string {
|
||||
ret := []string{}
|
||||
for _, ts := range x {
|
||||
ret = append(ret, fmt.Sprintf("%04.4d,%s", ts.hhmm, ts.bandwidth.String()))
|
||||
}
|
||||
return strings.Join(ret, " ")
|
||||
}
|
||||
|
||||
// Set the bandwidth timetable.
|
||||
func (x *BwTimetable) Set(s string) error {
|
||||
// The timetable is formatted as:
|
||||
// "hh:mm,bandwidth hh:mm,banwidth..." ex: "10:00,10G 11:30,1G 18:00,off"
|
||||
// If only a single bandwidth identifier is provided, we assume constant bandwidth.
|
||||
|
||||
if len(s) == 0 {
|
||||
return errors.New("empty string")
|
||||
}
|
||||
// Single value without time specification.
|
||||
if !strings.Contains(s, " ") && !strings.Contains(s, ",") {
|
||||
ts := BwTimeSlot{}
|
||||
if err := ts.bandwidth.Set(s); err != nil {
|
||||
return err
|
||||
}
|
||||
ts.hhmm = 0
|
||||
*x = BwTimetable{ts}
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, tok := range strings.Split(s, " ") {
|
||||
tv := strings.Split(tok, ",")
|
||||
|
||||
// Format must be HH:MM,BW
|
||||
if len(tv) != 2 {
|
||||
return errors.Errorf("invalid time/bandwidth specification: %q", tok)
|
||||
}
|
||||
|
||||
// Basic timespec sanity checking
|
||||
hhmm := tv[0]
|
||||
if len(hhmm) != 5 {
|
||||
return errors.Errorf("invalid time specification (hh:mm): %q", hhmm)
|
||||
}
|
||||
hh, err := strconv.Atoi(hhmm[0:2])
|
||||
if err != nil {
|
||||
return errors.Errorf("invalid hour in time specification %q: %v", hhmm, err)
|
||||
}
|
||||
if hh < 0 || hh > 23 {
|
||||
return errors.Errorf("invalid hour (must be between 00 and 23): %q", hh)
|
||||
}
|
||||
mm, err := strconv.Atoi(hhmm[3:])
|
||||
if err != nil {
|
||||
return errors.Errorf("invalid minute in time specification: %q: %v", hhmm, err)
|
||||
}
|
||||
if mm < 0 || mm > 59 {
|
||||
return errors.Errorf("invalid minute (must be between 00 and 59): %q", hh)
|
||||
}
|
||||
|
||||
ts := BwTimeSlot{
|
||||
hhmm: (hh * 100) + mm,
|
||||
}
|
||||
// Bandwidth limit for this time slot.
|
||||
if err := ts.bandwidth.Set(tv[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
*x = append(*x, ts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LimitAt returns a BwTimeSlot for the time requested.
|
||||
func (x BwTimetable) LimitAt(tt time.Time) BwTimeSlot {
|
||||
// If the timetable is empty, we return an unlimited BwTimeSlot starting at midnight.
|
||||
if len(x) == 0 {
|
||||
return BwTimeSlot{hhmm: 0, bandwidth: -1}
|
||||
}
|
||||
|
||||
hhmm := tt.Hour()*100 + tt.Minute()
|
||||
|
||||
// By default, we return the last element in the timetable. This
|
||||
// satisfies two conditions: 1) If there's only one element it
|
||||
// will always be selected, and 2) The last element of the table
|
||||
// will "wrap around" until overriden by an earlier time slot.
|
||||
// there's only one time slot in the timetable.
|
||||
ret := x[len(x)-1]
|
||||
|
||||
mindif := 0
|
||||
first := true
|
||||
|
||||
// Look for most recent time slot.
|
||||
for _, ts := range x {
|
||||
// Ignore the past
|
||||
if hhmm < ts.hhmm {
|
||||
continue
|
||||
}
|
||||
dif := ((hhmm / 100 * 60) + (hhmm % 100)) - ((ts.hhmm / 100 * 60) + (ts.hhmm % 100))
|
||||
if first {
|
||||
mindif = dif
|
||||
first = false
|
||||
}
|
||||
if dif <= mindif {
|
||||
mindif = dif
|
||||
ret = ts
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Type of the value
|
||||
func (x BwTimetable) Type() string {
|
||||
return "BwTimetable"
|
||||
}
|
||||
|
||||
// Check it satisfies the interface
|
||||
var _ pflag.Value = (*BwTimetable)(nil)
|
||||
|
||||
// optionToEnv converts an option name, eg "ignore-size" into an
|
||||
// environment name "RCLONE_IGNORE_SIZE"
|
||||
func optionToEnv(name string) string {
|
||||
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
|
||||
}
|
||||
|
||||
// setDefaultFromEnv constructs a name from the flag passed in and
|
||||
// sets the default from the environment if possible.
|
||||
func setDefaultFromEnv(name string) {
|
||||
key := optionToEnv(name)
|
||||
newValue, found := os.LookupEnv(key)
|
||||
if found {
|
||||
flag := pflag.Lookup(name)
|
||||
if flag == nil {
|
||||
log.Fatalf("Couldn't find flag %q", name)
|
||||
}
|
||||
err := flag.Value.Set(newValue)
|
||||
if err != nil {
|
||||
log.Fatalf("Invalid value for environment variable %q: %v", key, err)
|
||||
}
|
||||
// log.Printf("Set default for %q from %q to %q (%v)", name, key, newValue, flag.Value)
|
||||
flag.DefValue = newValue
|
||||
}
|
||||
}
|
||||
|
||||
// StringP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It is a thin wrapper around pflag.StringP
|
||||
func StringP(name, shorthand string, value string, usage string) (out *string) {
|
||||
out = pflag.StringP(name, shorthand, value, usage)
|
||||
setDefaultFromEnv(name)
|
||||
return out
|
||||
}
|
||||
|
||||
// BoolP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It is a thin wrapper around pflag.BoolP
|
||||
func BoolP(name, shorthand string, value bool, usage string) (out *bool) {
|
||||
out = pflag.BoolP(name, shorthand, value, usage)
|
||||
setDefaultFromEnv(name)
|
||||
return out
|
||||
}
|
||||
|
||||
// IntP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It is a thin wrapper around pflag.IntP
|
||||
func IntP(name, shorthand string, value int, usage string) (out *int) {
|
||||
out = pflag.IntP(name, shorthand, value, usage)
|
||||
setDefaultFromEnv(name)
|
||||
return out
|
||||
}
|
||||
|
||||
// DurationP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It is a thin wrapper around pflag.DurationP
|
||||
func DurationP(name, shorthand string, value time.Duration, usage string) (out *time.Duration) {
|
||||
out = pflag.DurationP(name, shorthand, value, usage)
|
||||
setDefaultFromEnv(name)
|
||||
return out
|
||||
}
|
||||
|
||||
// VarP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It is a thin wrapper around pflag.VarP
|
||||
func VarP(value pflag.Value, name, shorthand, usage string) {
|
||||
pflag.VarP(value, name, shorthand, usage)
|
||||
setDefaultFromEnv(name)
|
||||
}
|
||||
|
||||
// StringArrayP defines a flag which can be overridden by an environment variable
|
||||
//
|
||||
// It sets one value only - command line flags can be used to set more.
|
||||
//
|
||||
// It is a thin wrapper around pflag.StringArrayP
|
||||
func StringArrayP(name, shorthand string, value []string, usage string) (out *[]string) {
|
||||
out = pflag.StringArrayP(name, shorthand, value, usage)
|
||||
setDefaultFromEnv(name)
|
||||
return out
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSizeSuffixString(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in float64
|
||||
want string
|
||||
}{
|
||||
{0, "0"},
|
||||
{102, "102"},
|
||||
{1024, "1k"},
|
||||
{1024 * 1024, "1M"},
|
||||
{1024 * 1024 * 1024, "1G"},
|
||||
{10 * 1024 * 1024 * 1024, "10G"},
|
||||
{10.1 * 1024 * 1024 * 1024, "10.100G"},
|
||||
{-1, "off"},
|
||||
{-100, "off"},
|
||||
} {
|
||||
ss := SizeSuffix(test.in)
|
||||
got := ss.String()
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeSuffixUnit(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in float64
|
||||
want string
|
||||
}{
|
||||
{0, "0 Bytes"},
|
||||
{102, "102 Bytes"},
|
||||
{1024, "1 kBytes"},
|
||||
{1024 * 1024, "1 MBytes"},
|
||||
{1024 * 1024 * 1024, "1 GBytes"},
|
||||
{10 * 1024 * 1024 * 1024, "10 GBytes"},
|
||||
{10.1 * 1024 * 1024 * 1024, "10.100 GBytes"},
|
||||
{-1, "off"},
|
||||
{-100, "off"},
|
||||
} {
|
||||
ss := SizeSuffix(test.in)
|
||||
got := ss.Unit("Bytes")
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeSuffixSet(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want int64
|
||||
err bool
|
||||
}{
|
||||
{"0", 0, false},
|
||||
{"1b", 1, false},
|
||||
{"102B", 102, false},
|
||||
{"0.1k", 102, false},
|
||||
{"0.1", 102, false},
|
||||
{"1K", 1024, false},
|
||||
{"1", 1024, false},
|
||||
{"2.5", 1024 * 2.5, false},
|
||||
{"1M", 1024 * 1024, false},
|
||||
{"1.g", 1024 * 1024 * 1024, false},
|
||||
{"10G", 10 * 1024 * 1024 * 1024, false},
|
||||
{"off", -1, false},
|
||||
{"OFF", -1, false},
|
||||
{"", 0, true},
|
||||
{"1p", 0, true},
|
||||
{"1.p", 0, true},
|
||||
{"1p", 0, true},
|
||||
{"-1K", 0, true},
|
||||
} {
|
||||
ss := SizeSuffix(0)
|
||||
err := ss.Set(test.in)
|
||||
if test.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, test.want, int64(ss))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBwTimetableSet(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want BwTimetable
|
||||
err bool
|
||||
}{
|
||||
{"", BwTimetable{}, true},
|
||||
{"0", BwTimetable{BwTimeSlot{hhmm: 0, bandwidth: 0}}, false},
|
||||
{"666", BwTimetable{BwTimeSlot{hhmm: 0, bandwidth: 666 * 1024}}, false},
|
||||
{"10:20,666", BwTimetable{BwTimeSlot{hhmm: 1020, bandwidth: 666 * 1024}}, false},
|
||||
{
|
||||
"11:00,333 13:40,666 23:50,10M 23:59,off",
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1340, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: 10 * 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2359, bandwidth: -1},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{"bad,bad", BwTimetable{}, true},
|
||||
{"bad bad", BwTimetable{}, true},
|
||||
{"bad", BwTimetable{}, true},
|
||||
{"1000X", BwTimetable{}, true},
|
||||
{"2401,666", BwTimetable{}, true},
|
||||
{"1061,666", BwTimetable{}, true},
|
||||
} {
|
||||
tt := BwTimetable{}
|
||||
err := tt.Set(test.in)
|
||||
if test.err {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.Equal(t, test.want, tt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBwTimetableLimitAt(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
tt BwTimetable
|
||||
now time.Time
|
||||
want BwTimeSlot
|
||||
}{
|
||||
{
|
||||
BwTimetable{},
|
||||
time.Date(2017, time.April, 20, 15, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 0, bandwidth: -1},
|
||||
},
|
||||
{
|
||||
BwTimetable{BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024}},
|
||||
time.Date(2017, time.April, 20, 15, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 10, 15, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 11, 0, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 13, 1, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
},
|
||||
{
|
||||
BwTimetable{
|
||||
BwTimeSlot{hhmm: 1100, bandwidth: 333 * 1024},
|
||||
BwTimeSlot{hhmm: 1300, bandwidth: 666 * 1024},
|
||||
BwTimeSlot{hhmm: 2301, bandwidth: 1024 * 1024},
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
time.Date(2017, time.April, 20, 23, 59, 0, 0, time.UTC),
|
||||
BwTimeSlot{hhmm: 2350, bandwidth: -1},
|
||||
},
|
||||
} {
|
||||
slot := test.tt.LimitAt(test.now)
|
||||
assert.Equal(t, test.want, slot)
|
||||
}
|
||||
}
|
||||
|
|
@ -9,11 +9,10 @@ import (
|
|||
"syscall"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
var (
|
||||
oneFileSystem = pflag.BoolP("one-file-system", "x", false, "Don't cross filesystem boundaries.")
|
||||
oneFileSystem = fs.BoolP("one-file-system", "x", false, "Don't cross filesystem boundaries.")
|
||||
)
|
||||
|
||||
// readDevice turns a valid os.FileInfo into a device number,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"github.com/ncw/rclone/pacer"
|
||||
"github.com/ncw/rclone/rest"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
|
|
@ -75,8 +74,8 @@ func init() {
|
|||
Help: "Microsoft App Client Secret - leave blank normally.",
|
||||
}},
|
||||
})
|
||||
pflag.VarP(&chunkSize, "onedrive-chunk-size", "", "Above this size files will be chunked - must be multiple of 320k.")
|
||||
pflag.VarP(&uploadCutoff, "onedrive-upload-cutoff", "", "Cutoff for switching to chunked upload - must be <= 100MB")
|
||||
fs.VarP(&chunkSize, "onedrive-chunk-size", "", "Above this size files will be chunked - must be multiple of 320k.")
|
||||
fs.VarP(&uploadCutoff, "onedrive-upload-cutoff", "", "Cutoff for switching to chunked upload - must be <= 100MB")
|
||||
}
|
||||
|
||||
// Fs represents a remote one drive
|
||||
|
|
|
|||
5
s3/s3.go
5
s3/s3.go
|
|
@ -36,7 +36,6 @@ import (
|
|||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
|
|
@ -210,8 +209,8 @@ const (
|
|||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
s3ACL = pflag.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
|
||||
s3StorageClass = pflag.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA)")
|
||||
s3ACL = fs.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
|
||||
s3StorageClass = fs.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA)")
|
||||
)
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ import (
|
|||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/swift"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// Constants
|
||||
|
|
@ -83,7 +82,7 @@ func init() {
|
|||
},
|
||||
})
|
||||
// snet = flag.Bool("swift-snet", false, "Use internal service network") // FIXME not implemented
|
||||
pflag.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.")
|
||||
fs.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.")
|
||||
}
|
||||
|
||||
// Fs represents a remote swift server
|
||||
|
|
|
|||
Loading…
Reference in New Issue