support exclude file in --fast-list mode

fs/filter.go

@@ -406,22 +406,51 @@ func (f *Filter) ListContainsExcludeFile(entries DirEntries) bool {
 	return false
 }
 
-// IncludeDirectory returns whether this directory should be included
-// in the sync or not.
-func (f *Filter) IncludeDirectory(remote string) bool {
-	remote = strings.Trim(remote, "/")
-	// filesFrom takes precedence
-	if f.files != nil {
-		_, include := f.dirs[remote]
-		return include
+// IncludeDirectory returns a function which checks whether this
+// directory should be included in the sync or not.
+func (f *Filter) IncludeDirectory(fs Fs) func(string) (bool, error) {
+	return func(remote string) (bool, error) {
+		remote = strings.Trim(remote, "/")
+		// first check if we need to remove directory based on
+		// the exclude file
+		excl, err := f.DirContainsExcludeFile(fs, remote)
+		if err != nil {
+			return false, err
+		}
+		if excl {
+			return false, nil
+		}
+
+		// filesFrom takes precedence
+		if f.files != nil {
+			_, include := f.dirs[remote]
+			return include, nil
+		}
+		remote += "/"
+		for _, rule := range f.dirRules.rules {
+			if rule.Match(remote) {
+				return rule.Include, nil
+			}
+		}
+
+		return true, nil
 	}
-	remote += "/"
-	for _, rule := range f.dirRules.rules {
-		if rule.Match(remote) {
-			return rule.Include
+}
+
+// DirContainsExcludeFile checks if exclude file is present in a
+// directroy. If fs is nil, it works properly if ExcludeFile is an
+// empty string (for testing).
+func (f *Filter) DirContainsExcludeFile(fs Fs, remote string) (bool, error) {
+	if len(Config.Filter.ExcludeFile) > 0 {
+		exists, err := FileExists(fs, path.Join(remote, Config.Filter.ExcludeFile))
+		if err != nil {
+			return false, err
+		}
+		if exists {
+			return true, nil
 		}
 	}
-	return true
+	return false, nil
 }
 
 // Include returns whether this object should be included into the

fs/filter_test.go

@@ -164,7 +164,8 @@ type includeDirTest struct {
 
 func testDirInclude(t *testing.T, f *Filter, tests []includeDirTest) {
 	for _, test := range tests {
-		got := f.IncludeDirectory(test.in)
+		got, err := f.IncludeDirectory(nil)(test.in)
+		require.NoError(t, err)
 		assert.Equal(t, test.want, got, test.in)
 	}
 }

fs/operations.go

@@ -622,17 +622,20 @@ func ListDirSorted(fs Fs, includeAll bool, dir string) (entries DirEntries, err
 	if err != nil {
 		return nil, err
 	}
+	// This should happen only if exclude files lives in the
+	// starting directory, otherwise ListDirSorted should not be
+	// called.
 	if !includeAll && Config.Filter.ListContainsExcludeFile(entries) {
-		Debugf(dir, "Excluded from sync (and deletion) based on exclude file")
+		Debugf(dir, "Excluded from sync (and deletion)")
 		return nil, nil
 	}
-	return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory)
+	return filterAndSortDir(entries, includeAll, dir, Config.Filter.IncludeObject, Config.Filter.IncludeDirectory(fs))
 }
 
 // filter (if required) and check the entries, then sort them
 func filterAndSortDir(entries DirEntries, includeAll bool, dir string,
 	IncludeObject func(o Object) bool,
-	IncludeDirectory func(remote string) bool) (newEntries DirEntries, err error) {
+	IncludeDirectory func(remote string) (bool, error)) (newEntries DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
 	prefix := ""
 	if dir != "" {
@@ -649,9 +652,15 @@ func filterAndSortDir(entries DirEntries, includeAll bool, dir string,
 				Debugf(x, "Excluded from sync (and deletion)")
 			}
 		case Directory:
-			if !includeAll && !IncludeDirectory(x.Remote()) {
-				ok = false
-				Debugf(x, "Excluded from sync (and deletion)")
+			if !includeAll {
+				include, err := IncludeDirectory(x.Remote())
+				if err != nil {
+					return nil, err
+				}
+				if !include {
+					ok = false
+					Debugf(x, "Excluded from sync (and deletion)")
+				}
 			}
 		default:
 			return nil, errors.Errorf("unknown object type %T", entry)

fs/operations_internal_test.go

@@ -22,8 +22,8 @@ func TestFilterAndSortIncludeAll(t *testing.T) {
 	includeObject := func(o Object) bool {
 		return o != oB
 	}
-	includeDirectory := func(remote string) bool {
-		return remote != "c"
+	includeDirectory := func(remote string) (bool, error) {
+		return remote != "c", nil
 	}
 	// no filter
 	newEntries, err := filterAndSortDir(entries, true, "", includeObject, includeDirectory)

fs/walk.go

@@ -349,6 +349,10 @@ func (dt DirTree) String() string {
 
 func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) {
 	dirs := make(DirTree)
+	// Entries can come in arbitrary order. We use toPrune to keep
+	// all directories to exclude later.
+	toPrune := make(map[string]bool)
+	includeDirectory := Config.Filter.IncludeDirectory(f)
 	var mu sync.Mutex
 	err := listR(startPath, func(entries DirEntries) error {
 		mu.Lock()
@@ -372,8 +376,21 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L
 				} else {
 					Debugf(x, "Excluded from sync (and deletion)")
 				}
+				// Check if we need to prune a directory later.
+				if !includeAll && len(Config.Filter.ExcludeFile) > 0 {
+					basename := path.Base(x.Remote())
+					if basename == Config.Filter.ExcludeFile {
+						excludeDir := parentDir(x.Remote())
+						toPrune[excludeDir] = true
+						Debugf(basename, "Excluded from sync (and deletion) based on exclude file")
+					}
+				}
 			case Directory:
-				if includeAll || Config.Filter.IncludeDirectory(x.Remote()) {
+				inc, err := includeDirectory(x.Remote())
+				if err != nil {
+					return err
+				}
+				if includeAll || inc {
 					if maxLevel < 0 || slashes <= maxLevel-1 {
 						if slashes == maxLevel-1 {
 							// Just add the object if at maxLevel
@@ -398,6 +415,10 @@ func walkRDirTree(f Fs, startPath string, includeAll bool, maxLevel int, listR L
 	if len(dirs) == 0 {
 		dirs[startPath] = nil
 	}
+	err = dirs.Prune(toPrune)
+	if err != nil {
+		return nil, err
+	}
 	dirs.Sort()
 	return dirs, nil
 }