yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Factor each commmand into its own package

s3-about
Nick Craig-Wood 2016-08-05 17:12:27 +01:00
parent 0a7b34eefc
commit e27b91ffb8
25 changed files with 855 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/all/all.go Normal file
View File

@ -0,0 +1,29 @@
// Package all imports all the commands
package all
import (
// Active commands
_ "github.com/ncw/rclone/cmd"
_ "github.com/ncw/rclone/cmd/authorize"
_ "github.com/ncw/rclone/cmd/check"
_ "github.com/ncw/rclone/cmd/cleanup"
_ "github.com/ncw/rclone/cmd/config"
_ "github.com/ncw/rclone/cmd/copy"
_ "github.com/ncw/rclone/cmd/dedupe"
_ "github.com/ncw/rclone/cmd/delete"
_ "github.com/ncw/rclone/cmd/genautocomplete"
_ "github.com/ncw/rclone/cmd/gendocs"
_ "github.com/ncw/rclone/cmd/ls"
_ "github.com/ncw/rclone/cmd/lsd"
_ "github.com/ncw/rclone/cmd/lsl"
_ "github.com/ncw/rclone/cmd/md5sum"
_ "github.com/ncw/rclone/cmd/memtest"
_ "github.com/ncw/rclone/cmd/mkdir"
_ "github.com/ncw/rclone/cmd/move"
_ "github.com/ncw/rclone/cmd/purge"
_ "github.com/ncw/rclone/cmd/rmdir"
_ "github.com/ncw/rclone/cmd/sha1sum"
_ "github.com/ncw/rclone/cmd/size"
_ "github.com/ncw/rclone/cmd/sync"
_ "github.com/ncw/rclone/cmd/version"
)

24
cmd/authorize/authorize.go Normal file
View File

@ -0,0 +1,24 @@
package authorize
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(authorizeCmd)
}
var authorizeCmd = &cobra.Command{
Use: "authorize",
Short: `Remote authorization.`,
Long: `
Remote authorization. Used to authorize a remote or headless
rclone from a machine with a browser - use as instructed by
rclone config.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 3, command, args)
fs.Authorize(args)
},
}

30
cmd/check/check.go Normal file
View File

@ -0,0 +1,30 @@
package check
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(checkCmd)
}
var checkCmd = &cobra.Command{
Use: "check source:path dest:path",
Short: `Checks the files in the source and destination match.`,
Long: `
Checks the files in the source and destination match. It
compares sizes and MD5SUMs and prints a report of files which
don't match. It doesn't alter the source or destination.
` + "`" + `--size-only` + "`" + ` may be used to only compare the sizes, not the MD5SUMs.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
cmd.Run(false, command, func() error {
return fs.Check(fdst, fsrc)
})
},
}

27
cmd/cleanup/cleanup.go Normal file
View File

@ -0,0 +1,27 @@
package cleanup
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(cleanupCmd)
}
var cleanupCmd = &cobra.Command{
Use: "cleanup remote:path",
Short: `Clean up the remote if possible`,
Long: `
Clean up the remote if possible. Empty the trash or delete old file
versions. Not supported by all remotes.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(true, command, func() error {
return fs.CleanUp(fsrc)
})
},
}

572
cmd/cmd.go
View File

@ -11,15 +11,11 @@ import (
"log"
"os"
"path"
"path/filepath"
"runtime"
"runtime/pprof"
"strings"
"sync"
"time"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"github.com/spf13/pflag"
"github.com/ncw/rclone/fs"
@ -34,7 +30,6 @@ var (
version bool
logFile = pflag.StringP("log-file", "", "", "Log everything to this file")
retries = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
dedupeMode = fs.DeduplicateInteractive
)
// Root is the main rclone command
@ -72,26 +67,28 @@ and configuration walkthroughs.
* http://rclone.org/
`,
Run: func(cmd *cobra.Command, args []string) {
if version {
showVersion()
os.Exit(0)
}
},
}
// runRoot implements the main rclone command with no subcommands
func runRoot(cmd *cobra.Command, args []string) {
if version {
ShowVersion()
os.Exit(0)
} else {
_ = Root.Usage()
fmt.Fprintf(os.Stderr, "Command not found.\n")
os.Exit(1)
}
}
func init() {
Root.Run = runRoot
Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
Root.AddCommand(copyCmd, syncCmd, moveCmd, lsCmd, lsdCmd,
lslCmd, md5sumCmd, sha1sumCmd, sizeCmd, mkdirCmd,
rmdirCmd, purgeCmd, deleteCmd, checkCmd, dedupeCmd,
genautocompleteCmd, gendocsCmd, configCmd, authorizeCmd,
cleanupCmd, memtestCmd, versionCmd)
dedupeCmd.Flags().VarP(&dedupeMode, "dedupe-mode", "", "Dedupe mode interactive|skip|first|newest|oldest|rename.")
cobra.OnInitialize(initConfig)
}
func showVersion() {
// ShowVersion prints the version to stdout
func ShowVersion() {
fmt.Printf("rclone %s\n", fs.Version)
}
@ -233,545 +230,6 @@ func startStats() chan struct{} {
return stopStats
}
// The commands
var copyCmd = &cobra.Command{
Use: "copy source:path dest:path",
Short: `Copy files from source to dest, skipping already copied`,
Long: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing by size and modification time or
MD5SUM. Doesn't delete files from the destination.
Note that it is always the contents of the directory that is synced,
not the directory so when source:path is a directory, it's the
contents of source:path that are copied, not the directory name and
contents.
If dest:path doesn't exist, it is created and the source:path contents
go there.
For example
rclone copy source:sourcepath dest:destpath
Let's say there are two files in sourcepath
sourcepath/one.txt
sourcepath/two.txt
This copies them to
destpath/one.txt
destpath/two.txt
Not to
destpath/sourcepath/one.txt
destpath/sourcepath/two.txt
If you are familiar with ` + "`" + `rsync` + "`" + `, rclone always works as if you had
written a trailing / - meaning "copy the contents of this directory".
This applies to all commands and whether you are talking about the
source or destination.
See the ` + "`" + `--no-traverse` + "`" + ` option for controlling whether rclone lists
the destination directory or not.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(2, 2, cmd, args)
fsrc, fdst := NewFsSrcDst(args)
Run(true, cmd, func() error {
return fs.CopyDir(fdst, fsrc)
})
},
}
var syncCmd = &cobra.Command{
Use: "sync source:path dest:path",
Short: `Make source and dest identical, modifying destination only.`,
Long: `
Sync the source to the destination, changing the destination
only. Doesn't transfer unchanged files, testing by size and
modification time or MD5SUM. Destination is updated to match
source, including deleting files if necessary.
**Important**: Since this can cause data loss, test first with the
` + "`" + `--dry-run` + "`" + ` flag to see exactly what would be copied and deleted.
Note that files in the destination won't be deleted if there were any
errors at any point.
It is always the contents of the directory that is synced, not the
directory so when source:path is a directory, it's the contents of
source:path that are copied, not the directory name and contents. See
extended explanation in the ` + "`" + `copy` + "`" + ` command above if unsure.
If dest:path doesn't exist, it is created and the source:path contents
go there.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(2, 2, cmd, args)
fsrc, fdst := NewFsSrcDst(args)
Run(true, cmd, func() error {
return fs.Sync(fdst, fsrc)
})
},
}
var moveCmd = &cobra.Command{
Use: "move source:path dest:path",
Short: `Move files from source to dest.`,
Long: `
Moves the contents of the source directory to the destination
directory. Rclone will error if the source and destination overlap.
If no filters are in use and if possible this will server side move
` + "`" + `source:path` + "`" + ` into ` + "`" + `dest:path` + "`" + `. After this ` + "`" + `source:path` + "`" + ` will no
longer longer exist.
Otherwise for each file in ` + "`" + `source:path` + "`" + ` selected by the filters (if
any) this will move it into ` + "`" + `dest:path` + "`" + `. If possible a server side
move will be used, otherwise it will copy it (server side if possible)
into ` + "`" + `dest:path` + "`" + ` then delete the original (if no errors on copy) in
` + "`" + `source:path` + "`" + `.
**Important**: Since this can cause data loss, test first with the
--dry-run flag.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(2, 2, cmd, args)
fsrc, fdst := NewFsSrcDst(args)
Run(true, cmd, func() error {
return fs.MoveDir(fdst, fsrc)
})
},
}
var lsCmd = &cobra.Command{
Use: "ls remote:path",
Short: `List all the objects in the the path with size and path.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.List(fsrc, os.Stdout)
})
},
}
var lsdCmd = &cobra.Command{
Use: "lsd remote:path",
Short: `List all directories/containers/buckets in the the path.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.ListDir(fsrc, os.Stdout)
})
},
}
var lslCmd = &cobra.Command{
Use: "lsl remote:path",
Short: `List all the objects path with modification time, size and path.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.ListLong(fsrc, os.Stdout)
})
},
}
var md5sumCmd = &cobra.Command{
Use: "md5sum remote:path",
Short: `Produces an md5sum file for all the objects in the path.`,
Long: `
Produces an md5sum file for all the objects in the path. This
is in the same format as the standard md5sum tool produces.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.Md5sum(fsrc, os.Stdout)
})
},
}
var sha1sumCmd = &cobra.Command{
Use: "sha1sum remote:path",
Short: `Produces an sha1sum file for all the objects in the path.`,
Long: `
Produces an sha1sum file for all the objects in the path. This
is in the same format as the standard sha1sum tool produces.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.Sha1sum(fsrc, os.Stdout)
})
},
}
var sizeCmd = &cobra.Command{
Use: "size remote:path",
Short: `Prints the total size and number of objects in remote:path.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
objects, size, err := fs.Count(fsrc)
if err != nil {
return err
}
fmt.Printf("Total objects: %d\n", objects)
fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(size).Unit("Bytes"), size)
return nil
})
},
}
var mkdirCmd = &cobra.Command{
Use: "mkdir remote:path",
Short: `Make the path if it doesn't already exist.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fdst := NewFsDst(args)
Run(true, cmd, func() error {
return fs.Mkdir(fdst)
})
},
}
var rmdirCmd = &cobra.Command{
Use: "rmdir remote:path",
Short: `Remove the path if empty.`,
Long: `
Remove the path. Note that you can't remove a path with
objects in it, use purge for that.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fdst := NewFsDst(args)
Run(true, cmd, func() error {
return fs.Rmdir(fdst)
})
},
}
var purgeCmd = &cobra.Command{
Use: "purge remote:path",
Short: `Remove the path and all of its contents.`,
Long: `
Remove the path and all of its contents. Note that this does not obey
include/exclude filters - everything will be removed. Use ` + "`" + `delete` + "`" + ` if
you want to selectively delete files.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fdst := NewFsDst(args)
Run(true, cmd, func() error {
return fs.Purge(fdst)
})
},
}
var deleteCmd = &cobra.Command{
Use: "delete remote:path",
Short: `Remove the contents of path.`,
Long: `
Remove the contents of path. Unlike ` + "`" + `purge` + "`" + ` it obeys include/exclude
filters so can be used to selectively delete files.
Eg delete all files bigger than 100MBytes
Check what would be deleted first (use either)
rclone --min-size 100M lsl remote:path
rclone --dry-run --min-size 100M delete remote:path
Then delete
rclone --min-size 100M delete remote:path
That reads "delete everything with a minimum size of 100 MB", hence
delete all files bigger than 100MBytes.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(true, cmd, func() error {
return fs.Delete(fsrc)
})
},
}
var checkCmd = &cobra.Command{
Use: "check source:path dest:path",
Short: `Checks the files in the source and destination match.`,
Long: `
Checks the files in the source and destination match. It
compares sizes and MD5SUMs and prints a report of files which
don't match. It doesn't alter the source or destination.
` + "`" + `--size-only` + "`" + ` may be used to only compare the sizes, not the MD5SUMs.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(2, 2, cmd, args)
fsrc, fdst := NewFsSrcDst(args)
Run(false, cmd, func() error {
return fs.Check(fdst, fsrc)
})
},
}
var dedupeCmd = &cobra.Command{
Use: "dedupe [mode] remote:path",
Short: `Interactively find duplicate files delete/rename them.`,
Long: `
By default ` + "`" + `dedup` + "`" + ` interactively finds duplicate files and offers to
delete all but one or rename them to be different. Only useful with
Google Drive which can have duplicate file names.
The ` + "`" + `dedupe` + "`" + ` command will delete all but one of any identical (same
md5sum) files it finds without confirmation. This means that for most
duplicated files the ` + "`" + `dedupe` + "`" + ` command will not be interactive. You
can use ` + "`" + `--dry-run` + "`" + ` to see what would happen without doing anything.
Here is an example run.
Before - with duplicates
$ rclone lsl drive:dupes
6048320 2016-03-05 16:23:16.798000000 one.txt
6048320 2016-03-05 16:23:11.775000000 one.txt
564374 2016-03-05 16:23:06.731000000 one.txt
6048320 2016-03-05 16:18:26.092000000 one.txt
6048320 2016-03-05 16:22:46.185000000 two.txt
1744073 2016-03-05 16:22:38.104000000 two.txt
564374 2016-03-05 16:22:52.118000000 two.txt
Now the ` + "`" + `dedupe` + "`" + ` session
$ rclone dedupe drive:dupes
2016/03/05 16:24:37 Google drive root 'dupes': Looking for duplicates using interactive mode.
one.txt: Found 4 duplicates - deleting identical copies
one.txt: Deleting 2/3 identical duplicates (md5sum "1eedaa9fe86fd4b8632e2ac549403b36")
one.txt: 2 duplicates remain
1: 6048320 bytes, 2016-03-05 16:23:16.798000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
2: 564374 bytes, 2016-03-05 16:23:06.731000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
s) Skip and do nothing
k) Keep just one (choose which in next step)
r) Rename all to be different (by changing file.jpg to file-1.jpg)
s/k/r> k
Enter the number of the file to keep> 1
one.txt: Deleted 1 extra copies
two.txt: Found 3 duplicates - deleting identical copies
two.txt: 3 duplicates remain
1: 564374 bytes, 2016-03-05 16:22:52.118000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
2: 6048320 bytes, 2016-03-05 16:22:46.185000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
3: 1744073 bytes, 2016-03-05 16:22:38.104000000, md5sum 851957f7fb6f0bc4ce76be966d336802
s) Skip and do nothing
k) Keep just one (choose which in next step)
r) Rename all to be different (by changing file.jpg to file-1.jpg)
s/k/r> r
two-1.txt: renamed from: two.txt
two-2.txt: renamed from: two.txt
two-3.txt: renamed from: two.txt
The result being
$ rclone lsl drive:dupes
6048320 2016-03-05 16:23:16.798000000 one.txt
564374 2016-03-05 16:22:52.118000000 two-1.txt
6048320 2016-03-05 16:22:46.185000000 two-2.txt
1744073 2016-03-05 16:22:38.104000000 two-3.txt
Dedupe can be run non interactively using the ` + "`" + `--dedupe-mode` + "`" + ` flag or by using an extra parameter with the same value
* ` + "`" + `--dedupe-mode interactive` + "`" + ` - interactive as above.
* ` + "`" + `--dedupe-mode skip` + "`" + ` - removes identical files then skips anything left.
* ` + "`" + `--dedupe-mode first` + "`" + ` - removes identical files then keeps the first one.
* ` + "`" + `--dedupe-mode newest` + "`" + ` - removes identical files then keeps the newest one.
* ` + "`" + `--dedupe-mode oldest` + "`" + ` - removes identical files then keeps the oldest one.
* ` + "`" + `--dedupe-mode rename` + "`" + ` - removes identical files then renames the rest to be different.
For example to rename all the identically named photos in your Google Photos directory, do
rclone dedupe --dedupe-mode rename "drive:Google Photos"
Or
rclone dedupe rename "drive:Google Photos"
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 2, cmd, args)
if len(args) > 1 {
err := dedupeMode.Set(args[0])
if err != nil {
log.Fatal(err)
}
args = args[1:]
}
fdst := NewFsSrc(args)
Run(false, cmd, func() error {
return fs.Deduplicate(fdst, dedupeMode)
})
},
}
var configCmd = &cobra.Command{
Use: "config",
Short: `Enter an interactive configuration session.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(0, 0, cmd, args)
fs.EditConfig()
},
}
var genautocompleteCmd = &cobra.Command{
Use: "genautocomplete [output_file]",
Short: `Output bash completion script for rclone.`,
Long: `
Generates a bash shell autocompletion script for rclone.
This writes to /etc/bash_completion.d/rclone by default so will
probably need to be run with sudo or as root, eg
sudo rclone genautocomplete
Logout and login again to use the autocompletion scripts, or source
them directly
. /etc/bash_completion
If you supply a command line argument the script will be written
there.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(0, 1, cmd, args)
out := "/etc/bash_completion.d/rclone"
if len(args) > 0 {
out = args[0]
}
err := Root.GenBashCompletionFile(out)
if err != nil {
log.Fatal(err)
}
},
}
const gendocFrontmatterTemplate = `---
date: %s
title: "%s"
slug: %s
url: %s
---
`
var gendocsCmd = &cobra.Command{
Use: "gendocs output_directory",
Short: `Output markdown docs for rclone to the directory supplied.`,
Long: `
This produces markdown docs for the rclone commands to the directory
supplied. These are in a format suitable for hugo to render into the
rclone.org website.`,
RunE: func(cmd *cobra.Command, args []string) error {
CheckArgs(1, 1, cmd, args)
out := args[0]
err := os.MkdirAll(out, 0777)
if err != nil {
return err
}
now := time.Now().Format(time.RFC3339)
prepender := func(filename string) string {
name := filepath.Base(filename)
base := strings.TrimSuffix(name, path.Ext(name))
url := "/commands/" + strings.ToLower(base) + "/"
return fmt.Sprintf(gendocFrontmatterTemplate, now, strings.Replace(base, "_", " ", -1), base, url)
}
linkHandler := func(name string) string {
base := strings.TrimSuffix(name, path.Ext(name))
return "/commands/" + strings.ToLower(base) + "/"
}
return doc.GenMarkdownTreeCustom(Root, out, prepender, linkHandler)
},
}
var authorizeCmd = &cobra.Command{
Use: "authorize",
Short: `Remote authorization.`,
Long: `
Remote authorization. Used to authorize a remote or headless
rclone from a machine with a browser - use as instructed by
rclone config.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 3, cmd, args)
fs.Authorize(args)
},
}
var cleanupCmd = &cobra.Command{
Use: "cleanup remote:path",
Short: `Clean up the remote if possible`,
Long: `
Clean up the remote if possible. Empty the trash or delete old file
versions. Not supported by all remotes.
`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(true, cmd, func() error {
return fs.CleanUp(fsrc)
})
},
}
var memtestCmd = &cobra.Command{
Use: "memtest remote:path",
Short: `Load all the objects at remote:path and report memory stats.`,
Hidden: true,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(1, 1, cmd, args)
fsrc := NewFsSrc(args)
Run(false, cmd, func() error {
objects, _, err := fs.Count(fsrc)
if err != nil {
return err
}
objs := make([]fs.Object, 0, objects)
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
var mu sync.Mutex
err = fs.ListFn(fsrc, func(o fs.Object) {
mu.Lock()
objs = append(objs, o)
mu.Unlock()
})
if err != nil {
return err
}
runtime.GC()
runtime.ReadMemStats(&after)
usedMemory := after.Alloc - before.Alloc
fs.Log(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), usedMemory, float64(usedMemory)/float64(len(objs)))
fs.Log(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, after.Sys-before.Sys)
return nil
})
},
}
var versionCmd = &cobra.Command{
Use: "version",
Short: `Show the version number.`,
Run: func(cmd *cobra.Command, args []string) {
CheckArgs(0, 0, cmd, args)
showVersion()
},
}
// initConfig is run by cobra after initialising the flags
func initConfig() {
// Log file output

20
cmd/config/config.go Normal file
View File

@ -0,0 +1,20 @@
package config
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(configCmd)
}
var configCmd = &cobra.Command{
Use: "config",
Short: `Enter an interactive configuration session.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 0, command, args)
fs.EditConfig()
},
}

63
cmd/copy/copy.go Normal file
View File

@ -0,0 +1,63 @@
package copy
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(copyCmd)
}
var copyCmd = &cobra.Command{
Use: "copy source:path dest:path",
Short: `Copy files from source to dest, skipping already copied`,
Long: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing by size and modification time or
MD5SUM. Doesn't delete files from the destination.
Note that it is always the contents of the directory that is synced,
not the directory so when source:path is a directory, it's the
contents of source:path that are copied, not the directory name and
contents.
If dest:path doesn't exist, it is created and the source:path contents
go there.
For example
rclone copy source:sourcepath dest:destpath
Let's say there are two files in sourcepath
sourcepath/one.txt
sourcepath/two.txt
This copies them to
destpath/one.txt
destpath/two.txt
Not to
destpath/sourcepath/one.txt
destpath/sourcepath/two.txt
If you are familiar with ` + "`" + `rsync` + "`" + `, rclone always works as if you had
written a trailing / - meaning "copy the contents of this directory".
This applies to all commands and whether you are talking about the
source or destination.
See the ` + "`" + `--no-traverse` + "`" + ` option for controlling whether rclone lists
the destination directory or not.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
cmd.Run(true, command, func() error {
return fs.CopyDir(fdst, fsrc)
})
},
}

113
cmd/dedupe/dedupe.go Normal file
View File

@ -0,0 +1,113 @@
package dedupe
import (
"log"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
var (
dedupeMode = fs.DeduplicateInteractive
)
func init() {
cmd.Root.AddCommand(dedupeCmd)
dedupeCmd.Flags().VarP(&dedupeMode, "dedupe-mode", "", "Dedupe mode interactive|skip|first|newest|oldest|rename.")
}
var dedupeCmd = &cobra.Command{
Use: "dedupe [mode] remote:path",
Short: `Interactively find duplicate files delete/rename them.`,
Long: `
By default ` + "`" + `dedup` + "`" + ` interactively finds duplicate files and offers to
delete all but one or rename them to be different. Only useful with
Google Drive which can have duplicate file names.
The ` + "`" + `dedupe` + "`" + ` command will delete all but one of any identical (same
md5sum) files it finds without confirmation. This means that for most
duplicated files the ` + "`" + `dedupe` + "`" + ` command will not be interactive. You
can use ` + "`" + `--dry-run` + "`" + ` to see what would happen without doing anything.
Here is an example run.
Before - with duplicates
$ rclone lsl drive:dupes
6048320 2016-03-05 16:23:16.798000000 one.txt
6048320 2016-03-05 16:23:11.775000000 one.txt
564374 2016-03-05 16:23:06.731000000 one.txt
6048320 2016-03-05 16:18:26.092000000 one.txt
6048320 2016-03-05 16:22:46.185000000 two.txt
1744073 2016-03-05 16:22:38.104000000 two.txt
564374 2016-03-05 16:22:52.118000000 two.txt
Now the ` + "`" + `dedupe` + "`" + ` session
$ rclone dedupe drive:dupes
2016/03/05 16:24:37 Google drive root 'dupes': Looking for duplicates using interactive mode.
one.txt: Found 4 duplicates - deleting identical copies
one.txt: Deleting 2/3 identical duplicates (md5sum "1eedaa9fe86fd4b8632e2ac549403b36")
one.txt: 2 duplicates remain
1: 6048320 bytes, 2016-03-05 16:23:16.798000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
2: 564374 bytes, 2016-03-05 16:23:06.731000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
s) Skip and do nothing
k) Keep just one (choose which in next step)
r) Rename all to be different (by changing file.jpg to file-1.jpg)
s/k/r> k
Enter the number of the file to keep> 1
one.txt: Deleted 1 extra copies
two.txt: Found 3 duplicates - deleting identical copies
two.txt: 3 duplicates remain
1: 564374 bytes, 2016-03-05 16:22:52.118000000, md5sum 7594e7dc9fc28f727c42ee3e0749de81
2: 6048320 bytes, 2016-03-05 16:22:46.185000000, md5sum 1eedaa9fe86fd4b8632e2ac549403b36
3: 1744073 bytes, 2016-03-05 16:22:38.104000000, md5sum 851957f7fb6f0bc4ce76be966d336802
s) Skip and do nothing
k) Keep just one (choose which in next step)
r) Rename all to be different (by changing file.jpg to file-1.jpg)
s/k/r> r
two-1.txt: renamed from: two.txt
two-2.txt: renamed from: two.txt
two-3.txt: renamed from: two.txt
The result being
$ rclone lsl drive:dupes
6048320 2016-03-05 16:23:16.798000000 one.txt
564374 2016-03-05 16:22:52.118000000 two-1.txt
6048320 2016-03-05 16:22:46.185000000 two-2.txt
1744073 2016-03-05 16:22:38.104000000 two-3.txt
Dedupe can be run non interactively using the ` + "`" + `--dedupe-mode` + "`" + ` flag or by using an extra parameter with the same value
* ` + "`" + `--dedupe-mode interactive` + "`" + ` - interactive as above.
* ` + "`" + `--dedupe-mode skip` + "`" + ` - removes identical files then skips anything left.
* ` + "`" + `--dedupe-mode first` + "`" + ` - removes identical files then keeps the first one.
* ` + "`" + `--dedupe-mode newest` + "`" + ` - removes identical files then keeps the newest one.
* ` + "`" + `--dedupe-mode oldest` + "`" + ` - removes identical files then keeps the oldest one.
* ` + "`" + `--dedupe-mode rename` + "`" + ` - removes identical files then renames the rest to be different.
For example to rename all the identically named photos in your Google Photos directory, do
rclone dedupe --dedupe-mode rename "drive:Google Photos"
Or
rclone dedupe rename "drive:Google Photos"
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 2, command, args)
if len(args) > 1 {
err := dedupeMode.Set(args[0])
if err != nil {
log.Fatal(err)
}
args = args[1:]
}
fdst := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.Deduplicate(fdst, dedupeMode)
})
},
}

41
cmd/delete/delete.go Normal file
View File

@ -0,0 +1,41 @@
package delete
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(deleteCmd)
}
var deleteCmd = &cobra.Command{
Use: "delete remote:path",
Short: `Remove the contents of path.`,
Long: `
Remove the contents of path. Unlike ` + "`" + `purge` + "`" + ` it obeys include/exclude
filters so can be used to selectively delete files.
Eg delete all files bigger than 100MBytes
Check what would be deleted first (use either)
rclone --min-size 100M lsl remote:path
rclone --dry-run --min-size 100M delete remote:path
Then delete
rclone --min-size 100M delete remote:path
That reads "delete everything with a minimum size of 100 MB", hence
delete all files bigger than 100MBytes.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(true, command, func() error {
return fs.Delete(fsrc)
})
},
}

44
cmd/genautocomplete/genautocomplete.go Normal file
View File

@ -0,0 +1,44 @@
package genautocomplete
import (
"log"
"github.com/ncw/rclone/cmd"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(genautocompleteCmd)
}
var genautocompleteCmd = &cobra.Command{
Use: "genautocomplete [output_file]",
Short: `Output bash completion script for rclone.`,
Long: `
Generates a bash shell autocompletion script for rclone.
This writes to /etc/bash_completion.d/rclone by default so will
probably need to be run with sudo or as root, eg
sudo rclone genautocomplete
Logout and login again to use the autocompletion scripts, or source
them directly
. /etc/bash_completion
If you supply a command line argument the script will be written
there.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 1, command, args)
out := "/etc/bash_completion.d/rclone"
if len(args) > 0 {
out = args[0]
}
err := cmd.Root.GenBashCompletionFile(out)
if err != nil {
log.Fatal(err)
}
},
}

55
cmd/gendocs/gendocs.go Normal file
View File

@ -0,0 +1,55 @@
package gendocs
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/ncw/rclone/cmd"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
)
func init() {
cmd.Root.AddCommand(gendocsCmd)
}
const gendocFrontmatterTemplate = `---
date: %s
title: "%s"
slug: %s
url: %s
---
`
var gendocsCmd = &cobra.Command{
Use: "gendocs output_directory",
Short: `Output markdown docs for rclone to the directory supplied.`,
Long: `
This produces markdown docs for the rclone commands to the directory
supplied. These are in a format suitable for hugo to render into the
rclone.org website.`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(1, 1, command, args)
out := args[0]
err := os.MkdirAll(out, 0777)
if err != nil {
return err
}
now := time.Now().Format(time.RFC3339)
prepender := func(filename string) string {
name := filepath.Base(filename)
base := strings.TrimSuffix(name, path.Ext(name))
url := "/commands/" + strings.ToLower(base) + "/"
return fmt.Sprintf(gendocFrontmatterTemplate, now, strings.Replace(base, "_", " ", -1), base, url)
}
linkHandler := func(name string) string {
base := strings.TrimSuffix(name, path.Ext(name))
return "/commands/" + strings.ToLower(base) + "/"
}
return doc.GenMarkdownTreeCustom(cmd.Root, out, prepender, linkHandler)
},
}

25
cmd/ls/ls.go Normal file
View File

@ -0,0 +1,25 @@
package ls
import (
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(lsCmd)
}
var lsCmd = &cobra.Command{
Use: "ls remote:path",
Short: `List all the objects in the the path with size and path.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.List(fsrc, os.Stdout)
})
},
}

25
cmd/lsd/lsd.go Normal file
View File

@ -0,0 +1,25 @@
package lsd
import (
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(lsdCmd)
}
var lsdCmd = &cobra.Command{
Use: "lsd remote:path",
Short: `List all directories/containers/buckets in the the path.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.ListDir(fsrc, os.Stdout)
})
},
}

25
cmd/lsl/lsl.go Normal file
View File

@ -0,0 +1,25 @@
package lsl
import (
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(lslCmd)
}
var lslCmd = &cobra.Command{
Use: "lsl remote:path",
Short: `List all the objects path with modification time, size and path.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.ListLong(fsrc, os.Stdout)
})
},
}

29
cmd/md5sum/md5sum.go Normal file
View File

@ -0,0 +1,29 @@
package md5sum
import (
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(md5sumCmd)
}
var md5sumCmd = &cobra.Command{
Use: "md5sum remote:path",
Short: `Produces an md5sum file for all the objects in the path.`,
Long: `
Produces an md5sum file for all the objects in the path. This
is in the same format as the standard md5sum tool produces.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.Md5sum(fsrc, os.Stdout)
})
},
}

49
cmd/memtest/memtest.go Normal file
View File

@ -0,0 +1,49 @@
package memtest
import (
"runtime"
"sync"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(memtestCmd)
}
var memtestCmd = &cobra.Command{
Use: "memtest remote:path",
Short: `Load all the objects at remote:path and report memory stats.`,
Hidden: true,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
objects, _, err := fs.Count(fsrc)
if err != nil {
return err
}
objs := make([]fs.Object, 0, objects)
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
var mu sync.Mutex
err = fs.ListFn(fsrc, func(o fs.Object) {
mu.Lock()
objs = append(objs, o)
mu.Unlock()
})
if err != nil {
return err
}
runtime.GC()
runtime.ReadMemStats(&after)
usedMemory := after.Alloc - before.Alloc
fs.Log(nil, "%d objects took %d bytes, %.1f bytes/object", len(objs), usedMemory, float64(usedMemory)/float64(len(objs)))
fs.Log(nil, "System memory changed from %d to %d bytes a change of %d bytes", before.Sys, after.Sys, after.Sys-before.Sys)
return nil
})
},
}

23
cmd/mkdir/mkdir.go Normal file
View File

@ -0,0 +1,23 @@
package mkdir
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(mkdirCmd)
}
var mkdirCmd = &cobra.Command{
Use: "mkdir remote:path",
Short: `Make the path if it doesn't already exist.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fdst := cmd.NewFsDst(args)
cmd.Run(true, command, func() error {
return fs.Mkdir(fdst)
})
},
}

40
cmd/move/move.go Normal file
View File

@ -0,0 +1,40 @@
package move
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(moveCmd)
}
var moveCmd = &cobra.Command{
Use: "move source:path dest:path",
Short: `Move files from source to dest.`,
Long: `
Moves the contents of the source directory to the destination
directory. Rclone will error if the source and destination overlap.
If no filters are in use and if possible this will server side move
` + "`" + `source:path` + "`" + ` into ` + "`" + `dest:path` + "`" + `. After this ` + "`" + `source:path` + "`" + ` will no
longer longer exist.
Otherwise for each file in ` + "`" + `source:path` + "`" + ` selected by the filters (if
any) this will move it into ` + "`" + `dest:path` + "`" + `. If possible a server side
move will be used, otherwise it will copy it (server side if possible)
into ` + "`" + `dest:path` + "`" + ` then delete the original (if no errors on copy) in
` + "`" + `source:path` + "`" + `.
**Important**: Since this can cause data loss, test first with the
--dry-run flag.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
cmd.Run(true, command, func() error {
return fs.MoveDir(fdst, fsrc)
})
},
}

28
cmd/purge/purge.go Normal file
View File

@ -0,0 +1,28 @@
package purge
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(purgeCmd)
}
var purgeCmd = &cobra.Command{
Use: "purge remote:path",
Short: `Remove the path and all of its contents.`,
Long: `
Remove the path and all of its contents. Note that this does not obey
include/exclude filters - everything will be removed. Use ` + "`" + `delete` + "`" + ` if
you want to selectively delete files.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fdst := cmd.NewFsDst(args)
cmd.Run(true, command, func() error {
return fs.Purge(fdst)
})
},
}

26
cmd/rmdir/rmdir.go Normal file
View File

@ -0,0 +1,26 @@
package rmdir
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(rmdirCmd)
}
var rmdirCmd = &cobra.Command{
Use: "rmdir remote:path",
Short: `Remove the path if empty.`,
Long: `
Remove the path. Note that you can't remove a path with
objects in it, use purge for that.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fdst := cmd.NewFsDst(args)
cmd.Run(true, command, func() error {
return fs.Rmdir(fdst)
})
},
}

29
cmd/sha1sum/sha1sum.go Normal file
View File

@ -0,0 +1,29 @@
package sha1sum
import (
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(sha1sumCmd)
}
var sha1sumCmd = &cobra.Command{
Use: "sha1sum remote:path",
Short: `Produces an sha1sum file for all the objects in the path.`,
Long: `
Produces an sha1sum file for all the objects in the path. This
is in the same format as the standard sha1sum tool produces.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
return fs.Sha1sum(fsrc, os.Stdout)
})
},
}

31
cmd/size/size.go Normal file
View File

@ -0,0 +1,31 @@
package size
import (
"fmt"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(sizeCmd)
}
var sizeCmd = &cobra.Command{
Use: "size remote:path",
Short: `Prints the total size and number of objects in remote:path.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, command, func() error {
objects, size, err := fs.Count(fsrc)
if err != nil {
return err
}
fmt.Printf("Total objects: %d\n", objects)
fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(size).Unit("Bytes"), size)
return nil
})
},
}

43
cmd/sync/sync.go Normal file
View File

@ -0,0 +1,43 @@
package sync
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(syncCmd)
}
var syncCmd = &cobra.Command{
Use: "sync source:path dest:path",
Short: `Make source and dest identical, modifying destination only.`,
Long: `
Sync the source to the destination, changing the destination
only. Doesn't transfer unchanged files, testing by size and
modification time or MD5SUM. Destination is updated to match
source, including deleting files if necessary.
**Important**: Since this can cause data loss, test first with the
` + "`" + `--dry-run` + "`" + ` flag to see exactly what would be copied and deleted.
Note that files in the destination won't be deleted if there were any
errors at any point.
It is always the contents of the directory that is synced, not the
directory so when source:path is a directory, it's the contents of
source:path that are copied, not the directory name and contents. See
extended explanation in the ` + "`" + `copy` + "`" + ` command above if unsure.
If dest:path doesn't exist, it is created and the source:path contents
go there.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsrc, fdst := cmd.NewFsSrcDst(args)
cmd.Run(true, command, func() error {
return fs.Sync(fdst, fsrc)
})
},
}

19
cmd/version/version.go Normal file
View File

@ -0,0 +1,19 @@
package version
import (
"github.com/ncw/rclone/cmd"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(versionCmd)
}
var versionCmd = &cobra.Command{
Use: "version",
Short: `Show the version number.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 0, command, args)
cmd.ShowVersion()
},
}

3
rclone.go
View File

@ -8,7 +8,8 @@ import (
"os"
"github.com/ncw/rclone/cmd"
_ "github.com/ncw/rclone/fs/all" // import all fs
_ "github.com/ncw/rclone/cmd/all" // import all commands
_ "github.com/ncw/rclone/fs/all" // import all fs
)
func main() {