sync: add --no-check-dest flag - fixes #3616

docs/content/docs.md

@@ -770,6 +770,23 @@ in effect (the defaults):
 - 500MB..750MB files will be downloaded with 3 streams
 - 750MB+ files will be downloaded with 4 streams
 
+### --no-check-dest ###
+
+The `--no-check-dest` can be used with `move` or `copy` and it causes
+rclone not to check the destination at all when copying files.
+
+This means that:
+
+- the destination is not listed minimising the API calls
+- files are always transferred
+- this can cause duplicates on remotes which allow it (eg Google Drive)
+- `--retries 1` is recommended otherwise you'll transfer everything again on a retry
+
+This flag is useful to minimise the transactions if you know that none
+of the files are on the destination.
+
+This is a specialized flag which should be ignored by most users!
+
 ### --no-gzip-encoding ###
 
 Don't set `Accept-Encoding: gzip`.  This means that rclone won't ask

fs/config.go

@@ -67,6 +67,7 @@ type ConfigInfo struct {
 	IgnoreChecksum         bool
 	IgnoreCaseSync         bool
 	NoTraverse             bool
+	NoCheckDest            bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
 	CompareDest            string

fs/config/configflags/configflags.go

@@ -68,6 +68,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
 	flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
+	flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
 	flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.")
 	flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.")

fs/march/march.go

@@ -30,6 +30,7 @@ type March struct {
 	SrcIncludeAll bool            // don't include all files in the src
 	DstIncludeAll bool            // don't include all files in the destination
 	Callback      Marcher         // object to call with results
+	NoCheckDest   bool            // transfer all objects regardless without checking dst
 	// internal state
 	srcListDir listDirFn // function to call to list a directory in the src
 	dstListDir listDirFn // function to call to list a directory in the dst
@@ -188,6 +189,7 @@ func (m *March) Run() error {
 		srcDepth:  srcDepth - 1,
 		dstRemote: m.Dir,
 		dstDepth:  dstDepth - 1,
+		noDst:     m.NoCheckDest,
 	}
 	go func() {
 		// when the context is cancelled discard the remaining jobs
@@ -406,7 +408,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
 
 	// If NoTraverse is set, then try to find a matching object
 	// for each item in the srcList
-	if m.NoTraverse {
+	if m.NoTraverse && !m.NoCheckDest {
 		for _, src := range srcList {
 			if srcObj, ok := src.(fs.Object); ok {
 				leaf := path.Base(srcObj.Remote())

fs/operations/operations.go

@@ -1705,11 +1705,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 	}
 
 	// Find dst object if it exists
-	dstObj, err := fdst.NewObject(ctx, dstFileName)
-	if err == fs.ErrorObjectNotFound {
-		dstObj = nil
-	} else if err != nil {
-		return err
+	var dstObj fs.Object
+	if !fs.Config.NoCheckDest {
+		dstObj, err = fdst.NewObject(ctx, dstFileName)
+		if err == fs.ErrorObjectNotFound {
+			dstObj = nil
+		} else if err != nil {
+			return err
+		}
 	}
 
 	// Special case for changing case of a file on a case insensitive remote

fs/sync/sync.go

@@ -31,6 +31,7 @@ type syncCopyMove struct {
 	ctx             context.Context        // internal context for controlling go-routines
 	cancel          func()                 // cancel the context
 	noTraverse      bool                   // if set don't traverse the dst
+	noCheckDest     bool                   // if set transfer all objects regardless without checking dst
 	deletersWg      sync.WaitGroup         // for delete before go routine
 	deleteFilesCh   chan fs.Object         // channel to receive deletes if delete before
 	trackRenames    bool                   // set if we should do server side renames
@@ -82,6 +83,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		dstEmptyDirs:       make(map[string]fs.DirEntry),
 		srcEmptyDirs:       make(map[string]fs.DirEntry),
 		noTraverse:         fs.Config.NoTraverse,
+		noCheckDest:        fs.Config.NoCheckDest,
 		toBeChecked:        newPipe(accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog),
 		toBeUploaded:       newPipe(accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog),
 		deleteFilesCh:      make(chan fs.Object, fs.Config.Checkers),
@@ -95,6 +97,17 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		fs.Errorf(nil, "Ignoring --no-traverse with sync")
 		s.noTraverse = false
 	}
+	if s.noCheckDest {
+		if s.deleteMode != fs.DeleteModeOff {
+			return nil, errors.New("can't use --no-check-dest with sync: use copy instead")
+		}
+		if fs.Config.Immutable {
+			return nil, errors.New("can't use --no-check-dest with --immutable")
+		}
+		if s.backupDir != nil {
+			return nil, errors.New("can't use --no-check-dest with --backup-dir")
+		}
+	}
 	if s.trackRenames {
 		// Don't track renames for remotes without server-side move support.
 		if !operations.CanServerSideMove(fdst) {
@@ -667,6 +680,7 @@ func (s *syncCopyMove) run() error {
 		NoTraverse:    s.noTraverse,
 		Callback:      s,
 		DstIncludeAll: filter.Active.Opt.DeleteExcluded,
+		NoCheckDest:   s.noCheckDest,
 	}
 	s.processError(m.Run())