From eb0fc21533bc18780f781b9ecaf162e77604acd1 Mon Sep 17 00:00:00 2001 From: fishbullet Date: Sun, 15 Mar 2020 16:58:46 +0300 Subject: [PATCH] fs: filter flags ability to read from stdin - fixes #4034 --- docs/content/filtering.md | 3 ++- docs/content/flags.md | 8 ++++---- fs/filter/filter.go | 15 ++++++++++----- fs/filter/filter_test.go | 24 ++++++++++++++++++++++-- fs/filter/filterflags/filterflags.go | 8 ++++---- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/docs/content/filtering.md b/docs/content/filtering.md index a593ef908..96bcbfa89 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -17,7 +17,8 @@ Each path as it passes through rclone is matched against the include and exclude rules like `--include`, `--exclude`, `--include-from`, `--exclude-from`, `--filter`, or `--filter-from`. The simplest way to try them out is using the `ls` command, or `--dry-run` together with -`-v`. +`-v`. `--filter-from`, `--exclude-from`, `--include-from`, `--files-from` +understand `-` as a file name to mean read from standard input. ## Patterns ## diff --git a/docs/content/flags.md b/docs/content/flags.md index 85a5b460a..ded8129c8 100755 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -41,13 +41,13 @@ These flags are available for every command. --dump-bodies Dump HTTP headers and bodies - may contain sensitive info --dump-headers Dump HTTP headers - may contain sensitive info --exclude stringArray Exclude files matching pattern - --exclude-from stringArray Read exclude patterns from file + --exclude-from stringArray Read exclude patterns from file (use - to read from stdin). --exclude-if-present string Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available. Uses more memory but fewer transactions. - --files-from stringArray Read list of source-file names from file + --files-from stringArray Read list of source-file names from file (use - to read from stdin). -f, --filter stringArray Add a file-filtering rule - --filter-from stringArray Read filtering patterns from a file + --filter-from stringArray Read filtering patterns from a file (use - to read from stdin). --ignore-case Ignore case in filters (case insensitive) --ignore-case-sync Ignore case when synchronizing --ignore-checksum Skip post copy check of checksums. @@ -57,7 +57,7 @@ These flags are available for every command. -I, --ignore-times Don't skip files that match size and time - transfer all files --immutable Do not modify files. Fail if existing files have been modified. --include stringArray Include files matching pattern - --include-from stringArray Read include patterns from file + --include-from stringArray Read include patterns from file (use - to read from stdin). --log-file string Log everything to this file --log-format string Comma separated list of log format options (default "date,time") --log-level string Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE") diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 460fad6a6..aa52ab010 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -465,12 +465,17 @@ func (f *Filter) IncludeObject(ctx context.Context, o fs.Object) bool { // // It ignores empty lines and lines starting with '#' or ';' func forEachLine(path string, fn func(string) error) (err error) { - in, err := os.Open(path) - if err != nil { - return err + var scanner *bufio.Scanner + if path == "-" { + scanner = bufio.NewScanner(os.Stdin) + } else { + in, err := os.Open(path) + if err != nil { + return err + } + scanner = bufio.NewScanner(in) + defer fs.CheckClose(in, &err) } - defer fs.CheckClose(in, &err) - scanner := bufio.NewScanner(in) for scanner.Scan() { line := scanner.Text() line = strings.TrimSpace(line) diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 773b3eb04..0d1b4e90f 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -517,7 +517,7 @@ func TestFilterAddDirRuleOrFileRule(t *testing.T) { } } -func TestFilterForEachLine(t *testing.T) { +func testFilterForEachLine(t *testing.T, useStdin bool) { file := testFile(t, `; comment one # another comment @@ -534,7 +534,19 @@ five require.NoError(t, err) }() lines := []string{} - err := forEachLine(file, func(s string) error { + fileName := file + if useStdin { + in, err := os.Open(file) + require.NoError(t, err) + oldStdin := os.Stdin + os.Stdin = in + defer func() { + os.Stdin = oldStdin + _ = in.Close() + }() + fileName = "-" + } + err := forEachLine(fileName, func(s string) error { lines = append(lines, s) return nil }) @@ -542,6 +554,14 @@ five assert.Equal(t, "one,two,three,four,five,six", strings.Join(lines, ",")) } +func TestFilterForEachLine(t *testing.T) { + testFilterForEachLine(t, false) +} + +func TestFilterForEachLineStdin(t *testing.T) { + testFilterForEachLine(t, true) +} + func TestFilterMatchesFromDocs(t *testing.T) { for _, test := range []struct { glob string diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go index 913cf7581..603218b04 100644 --- a/fs/filter/filterflags/filterflags.go +++ b/fs/filter/filterflags/filterflags.go @@ -24,13 +24,13 @@ func AddFlags(flagSet *pflag.FlagSet) { rc.AddOptionReload("filter", &Opt, Reload) flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync") flags.StringArrayVarP(flagSet, &Opt.FilterRule, "filter", "f", nil, "Add a file-filtering rule") - flags.StringArrayVarP(flagSet, &Opt.FilterFrom, "filter-from", "", nil, "Read filtering patterns from a file") + flags.StringArrayVarP(flagSet, &Opt.FilterFrom, "filter-from", "", nil, "Read filtering patterns from a file (use - to read from stdin)") flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, "exclude", "", nil, "Exclude files matching pattern") - flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, "exclude-from", "", nil, "Read exclude patterns from file") + flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, "exclude-from", "", nil, "Read exclude patterns from file (use - to read from stdin)") flags.StringVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", "", "Exclude directories if filename is present") flags.StringArrayVarP(flagSet, &Opt.IncludeRule, "include", "", nil, "Include files matching pattern") - flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, "include-from", "", nil, "Read include patterns from file") - flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file") + flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, "include-from", "", nil, "Read include patterns from file (use - to read from stdin)") + flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)") flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y") flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y") flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in k or suffix b|k|M|G")