yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix golint warnings

s3-about
Nick Craig-Wood 2015-09-22 18:47:16 +01:00
parent 2ed158aba3
commit e9c915e6fe
24 changed files with 421 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -10,6 +10,11 @@ test: rclone
go test ./... go test ./...
cd fs && ./test_all.sh cd fs && ./test_all.sh
check: rclone
go vet ./...
errcheck ./...
golint ./...
doc: rclone.1 MANUAL.html MANUAL.txt doc: rclone.1 MANUAL.html MANUAL.txt
rclone.1: MANUAL.md rclone.1: MANUAL.md
@ -52,7 +57,7 @@ serve:
tag: tag:
@echo "Old tag is $(LAST_TAG)" @echo "Old tag is $(LAST_TAG)"
@echo "New tag is $(NEW_TAG)" @echo "New tag is $(NEW_TAG)"
echo -e "package fs\n const Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go echo -e "package fs\n\n// Version of rclone\nconst Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
perl -lpe 's/VERSION/${NEW_TAG}/g; s/DATE/'`date -I`'/g;' docs/content/downloads.md.in > docs/content/downloads.md perl -lpe 's/VERSION/${NEW_TAG}/g; s/DATE/'`date -I`'/g;' docs/content/downloads.md.in > docs/content/downloads.md
git tag $(NEW_TAG) git tag $(NEW_TAG)
@echo "Add this to changelog in docs/content/changelog.md" @echo "Add this to changelog in docs/content/changelog.md"

3
RELEASE.md
View File

@ -4,9 +4,12 @@ Required software for making a release
* Run `gox -build-toolchain` * Run `gox -build-toolchain`
* This assumes you have your own source checkout * This assumes you have your own source checkout
* pandoc for making the html and man pages * pandoc for making the html and man pages
* errcheck - go get github.com/kisielk/errcheck
* golint - go get github.com/golang/lint
Making a release Making a release
* go get -u -f -v ./... * go get -u -f -v ./...
* make check
* make test * make test
* make tag * make tag
* edit docs/content/changelog.md * edit docs/content/changelog.md

49
amazonclouddrive/amazonclouddrive.go
View File

@ -1,4 +1,5 @@
// Amazon Cloud Drive interface // Package amazonclouddrive provides an interface to the Amazon Cloud
// Drive object storage system.
package amazonclouddrive package amazonclouddrive
/* /*
@ -59,7 +60,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "amazon cloud drive", Name: "amazon cloud drive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -98,12 +99,12 @@ type FsObjectAcd struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsAcd) Name() string { func (f *FsAcd) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsAcd) Root() string { func (f *FsAcd) Root() string {
return f.root return f.root
} }
@ -242,17 +243,17 @@ func (f *FsAcd) newFsObjectWithInfo(remote string, info *acd.Node) fs.Object {
return o return o
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsAcd) NewFsObject(remote string) fs.Object { func (f *FsAcd) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// FindLeaf finds a directory of name leaf in the folder with ID pathId // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *FsAcd) FindLeaf(pathId, leaf string) (pathIdOut string, found bool, err error) { func (f *FsAcd) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
//fs.Debug(f, "FindLeaf(%q, %q)", pathId, leaf) //fs.Debug(f, "FindLeaf(%q, %q)", pathID, leaf)
folder := acd.FolderFromId(pathId, f.c.Nodes) folder := acd.FolderFromId(pathID, f.c.Nodes)
var resp *http.Response var resp *http.Response
var subFolder *acd.Folder var subFolder *acd.Folder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
@ -276,10 +277,10 @@ func (f *FsAcd) FindLeaf(pathId, leaf string) (pathIdOut string, found bool, err
return *subFolder.Id, true, nil return *subFolder.Id, true, nil
} }
// CreateDir makes a directory with pathId as parent and name leaf // CreateDir makes a directory with pathID as parent and name leaf
func (f *FsAcd) CreateDir(pathId, leaf string) (newId string, err error) { func (f *FsAcd) CreateDir(pathID, leaf string) (newID string, err error) {
//fmt.Printf("CreateDir(%q, %q)\n", pathId, leaf) //fmt.Printf("CreateDir(%q, %q)\n", pathID, leaf)
folder := acd.FolderFromId(pathId, f.c.Nodes) folder := acd.FolderFromId(pathID, f.c.Nodes)
var resp *http.Response var resp *http.Response
var info *acd.Folder var info *acd.Folder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
@ -305,8 +306,8 @@ type listAllFn func(*acd.Node) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *FsAcd) listAll(dirId string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *FsAcd) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
query := "parents:" + dirId query := "parents:" + dirID
if directoriesOnly { if directoriesOnly {
query += " AND kind:" + folderKind query += " AND kind:" + folderKind
} else if filesOnly { } else if filesOnly {
@ -358,11 +359,11 @@ OUTER:
// //
// This fetches the minimum amount of stuff but does more API calls // This fetches the minimum amount of stuff but does more API calls
// which makes it slow // which makes it slow
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { func (f *FsAcd) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup var wg sync.WaitGroup
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool { _, err := f.listAll(dirID, "", false, false, func(node *acd.Node) bool {
// Recurse on directories // Recurse on directories
switch *node.Kind { switch *node.Kind {
case folderKind: case folderKind:
@ -398,7 +399,7 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
return nil return nil
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsAcd) List() fs.ObjectsChan { func (f *FsAcd) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
@ -418,7 +419,7 @@ func (f *FsAcd) List() fs.ObjectsChan {
return out return out
} }
// Lists the containers // ListDir lists the directories
func (f *FsAcd) ListDir() fs.DirChan { func (f *FsAcd) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
@ -546,7 +547,7 @@ func (f *FsAcd) Rmdir() error {
return f.purgeCheck(true) return f.purgeCheck(true)
} }
// Return the precision // Precision return the precision of this Fs
func (f *FsAcd) Precision() time.Duration { func (f *FsAcd) Precision() time.Duration {
return fs.ModTimeNotSupported return fs.ModTimeNotSupported
} }
@ -585,7 +586,7 @@ func (f *FsAcd) Purge() error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectAcd) Fs() fs.Fs { func (o *FsObjectAcd) Fs() fs.Fs {
return o.acd return o.acd
} }
@ -598,7 +599,7 @@ func (o *FsObjectAcd) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectAcd) Remote() string { func (o *FsObjectAcd) Remote() string {
return o.remote return o.remote
} }
@ -661,13 +662,13 @@ func (o *FsObjectAcd) ModTime() time.Time {
return modTime return modTime
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectAcd) SetModTime(modTime time.Time) { func (o *FsObjectAcd) SetModTime(modTime time.Time) {
// FIXME not implemented // FIXME not implemented
return return
} }
// Is this object storable // Storable returns a boolean showing whether this object storable
func (o *FsObjectAcd) Storable() bool { func (o *FsObjectAcd) Storable() bool {
return true return true
} }

18
dircache/dircache.go
View File

@ -1,4 +1,4 @@
// dircache provides a simple cache for caching directory to path lookups // Package dircache provides a simple cache for caching directory to path lookups
package dircache package dircache
// _methods are called without the lock // _methods are called without the lock
@ -23,13 +23,13 @@ type DirCache struct {
foundRoot bool // Whether we have found the root or not foundRoot bool // Whether we have found the root or not
} }
// DirCache describes an interface for doing the low level directory work // DirCacher describes an interface for doing the low level directory work
type DirCacher interface { type DirCacher interface {
FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error)
CreateDir(pathID, leaf string) (newID string, err error) CreateDir(pathID, leaf string) (newID string, err error)
} }
// Make a new DirCache // New makes a DirCache
// //
// The cache is safe for concurrent use // The cache is safe for concurrent use
func New(root string, trueRootID string, fs DirCacher) *DirCache { func New(root string, trueRootID string, fs DirCacher) *DirCache {
@ -49,7 +49,7 @@ func (dc *DirCache) _get(path string) (id string, ok bool) {
return return
} }
// Gets an ID given a path // Get an ID given a path
func (dc *DirCache) Get(path string) (id string, ok bool) { func (dc *DirCache) Get(path string) (id string, ok bool) {
dc.mu.RLock() dc.mu.RLock()
id, ok = dc._get(path) id, ok = dc._get(path)
@ -91,7 +91,7 @@ func (dc *DirCache) Flush() {
dc.mu.Unlock() dc.mu.Unlock()
} }
// Splits a path into directory, leaf // SplitPath splits a path into directory, leaf
// //
// Path shouldn't start or end with a / // Path shouldn't start or end with a /
// //
@ -108,7 +108,8 @@ func SplitPath(path string) (directory, leaf string) {
return return
} }
// Finds the directory passed in returning the directory ID starting from pathID // FindDir finds the directory passed in returning the directory ID
// starting from pathID
// //
// Path shouldn't start or end with a / // Path shouldn't start or end with a /
// //
@ -207,7 +208,7 @@ func (dc *DirCache) FindPath(path string, create bool) (leaf, directoryID string
return return
} }
// Finds the root directory if not already found // FindRoot finds the root directory if not already found
// //
// Resets the root directory // Resets the root directory
// //
@ -268,7 +269,8 @@ func (dc *DirCache) RootParentID() (string, error) {
return dc.rootParentID, nil return dc.rootParentID, nil
} }
// Resets the root directory to the absolute root and clears the DirCache // ResetRoot resets the root directory to the absolute root and clears
// the DirCache
func (dc *DirCache) ResetRoot() { func (dc *DirCache) ResetRoot() {
dc.mu.Lock() dc.mu.Lock()
defer dc.mu.Unlock() defer dc.mu.Unlock()

94
drive/drive.go
View File

@ -1,4 +1,4 @@
// Drive interface // Package drive interfaces with the Google Drive object storage system
package drive package drive
// FIXME need to deal with some corner cases // FIXME need to deal with some corner cases
@ -61,7 +61,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "drive", Name: "drive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -106,12 +106,12 @@ type FsObjectDrive struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsDrive) Name() string { func (f *FsDrive) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsDrive) Root() string { func (f *FsDrive) Root() string {
return f.root return f.root
} }
@ -170,10 +170,10 @@ type listAllFn func(*drive.File) bool
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
// //
// Search params: https://developers.google.com/drive/search-parameters // Search params: https://developers.google.com/drive/search-parameters
func (f *FsDrive) listAll(dirId string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *FsDrive) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
query := fmt.Sprintf("trashed=false") query := fmt.Sprintf("trashed=false")
if dirId != "" { if dirID != "" {
query += fmt.Sprintf(" and '%s' in parents", dirId) query += fmt.Sprintf(" and '%s' in parents", dirID)
} }
if title != "" { if title != "" {
// Escaping the backslash isn't documented but seems to work // Escaping the backslash isn't documented but seems to work
@ -321,35 +321,35 @@ func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) fs.Object
return fs return fs
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDrive) NewFsObject(remote string) fs.Object { func (f *FsDrive) NewFsObject(remote string) fs.Object {
return f.newFsObjectWithInfo(remote, nil) return f.newFsObjectWithInfo(remote, nil)
} }
// FindLeaf finds a directory of name leaf in the folder with ID pathId // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *FsDrive) FindLeaf(pathId, leaf string) (pathIdOut string, found bool, err error) { func (f *FsDrive) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathId // Find the leaf in pathID
found, err = f.listAll(pathId, leaf, true, false, func(item *drive.File) bool { found, err = f.listAll(pathID, leaf, true, false, func(item *drive.File) bool {
if item.Title == leaf { if item.Title == leaf {
pathIdOut = item.Id pathIDOut = item.Id
return true return true
} }
return false return false
}) })
return pathIdOut, found, err return pathIDOut, found, err
} }
// CreateDir makes a directory with pathId as parent and name leaf // CreateDir makes a directory with pathID as parent and name leaf
func (f *FsDrive) CreateDir(pathId, leaf string) (newId string, err error) { func (f *FsDrive) CreateDir(pathID, leaf string) (newID string, err error) {
// fmt.Println("Making", path) // fmt.Println("Making", path)
// Define the metadata for the directory we are going to create. // Define the metadata for the directory we are going to create.
createInfo := &drive.File{ createInfo := &drive.File{
Title: leaf, Title: leaf,
Description: leaf, Description: leaf,
MimeType: driveFolderType, MimeType: driveFolderType,
Parents: []*drive.ParentReference{{Id: pathId}}, Parents: []*drive.ParentReference{{Id: pathID}},
} }
var info *drive.File var info *drive.File
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
@ -368,11 +368,11 @@ func (f *FsDrive) CreateDir(pathId, leaf string) (newId string, err error) {
// //
// This fetches the minimum amount of stuff but does more API calls // This fetches the minimum amount of stuff but does more API calls
// which makes it slow // which makes it slow
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { func (f *FsDrive) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup var wg sync.WaitGroup
_, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool { _, err := f.listAll(dirID, "", false, false, func(item *drive.File) bool {
// Recurse on directories // Recurse on directories
if item.MimeType == driveFolderType { if item.MimeType == driveFolderType {
wg.Add(1) wg.Add(1)
@ -388,7 +388,6 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan
} }
}() }()
return false
} else { } else {
// If item has no MD5 sum it isn't stored on drive, so ignore it // If item has no MD5 sum it isn't stored on drive, so ignore it
if item.Md5Checksum != "" { if item.Md5Checksum != "" {
@ -417,7 +416,7 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan
// //
// This is fast in terms of number of API calls, but slow in terms of // This is fast in terms of number of API calls, but slow in terms of
// fetching more data than it needs // fetching more data than it needs
func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) error { func (f *FsDrive) listDirFull(dirID string, path string, out fs.ObjectsChan) error {
// Orphans waiting for their parent // Orphans waiting for their parent
orphans := make(map[string][]*drive.File) orphans := make(map[string][]*drive.File)
@ -457,12 +456,12 @@ func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) err
// fmt.Printf("no parents %s %s: %#v\n", item.Title, item.Id, item) // fmt.Printf("no parents %s %s: %#v\n", item.Title, item.Id, item)
return false return false
} }
parentId := item.Parents[0].Id parentID := item.Parents[0].Id
directory, ok := f.dirCache.GetInv(parentId) directory, ok := f.dirCache.GetInv(parentID)
if !ok { if !ok {
// Haven't found the parent yet so add to orphans // Haven't found the parent yet so add to orphans
// fmt.Printf("orphan[%s] %s %s\n", parentId, item.Title, item.Id) // fmt.Printf("orphan[%s] %s %s\n", parentID, item.Title, item.Id)
orphans[parentId] = append(orphans[parentId], item) orphans[parentID] = append(orphans[parentID], item)
} else { } else {
outputItem(item, directory) outputItem(item, directory)
} }
@ -478,7 +477,7 @@ func (f *FsDrive) listDirFull(dirId string, path string, out fs.ObjectsChan) err
return nil return nil
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsDrive) List() fs.ObjectsChan { func (f *FsDrive) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
@ -502,7 +501,7 @@ func (f *FsDrive) List() fs.ObjectsChan {
return out return out
} }
// Walk the path returning a channel of FsObjects // ListDir walks the path returning a channel of directories
func (f *FsDrive) ListDir() fs.DirChan { func (f *FsDrive) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
@ -543,7 +542,7 @@ func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (
bytes: size, bytes: size,
} }
leaf, directoryId, err := f.dirCache.FindPath(remote, true) leaf, directoryID, err := f.dirCache.FindPath(remote, true)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -552,7 +551,7 @@ func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (
createInfo := &drive.File{ createInfo := &drive.File{
Title: leaf, Title: leaf,
Description: leaf, Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}}, Parents: []*drive.ParentReference{{Id: directoryID}},
MimeType: fs.MimeType(o), MimeType: fs.MimeType(o),
ModifiedDate: modTime.Format(timeFormatOut), ModifiedDate: modTime.Format(timeFormatOut),
} }
@ -638,8 +637,8 @@ func (f *FsDrive) Rmdir() error {
return nil return nil
} }
// Return the precision // Precision of the object storage system
func (fs *FsDrive) Precision() time.Duration { func (f *FsDrive) Precision() time.Duration {
return time.Millisecond return time.Millisecond
} }
@ -714,7 +713,7 @@ func (f *FsDrive) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (dstFs *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) { func (f *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDrive) srcObj, ok := src.(*FsObjectDrive)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
@ -722,13 +721,13 @@ func (dstFs *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) {
} }
// Temporary FsObject under construction // Temporary FsObject under construction
dstObj, dstInfo, err := dstFs.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes) dstObj, dstInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Do the move // Do the move
info, err := dstFs.svc.Files.Patch(srcObj.id, dstInfo).SetModifiedDate(true).Do() info, err := f.svc.Files.Patch(srcObj.id, dstInfo).SetModifiedDate(true).Do()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -737,14 +736,15 @@ func (dstFs *FsDrive) Move(src fs.Object, remote string) (fs.Object, error) {
return dstObj, nil return dstObj, nil
} }
// Move src to this remote using server side move operations. // DirMove moves src directory to this remote using server side move
// operations.
// //
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (dstFs *FsDrive) DirMove(src fs.Fs) error { func (f *FsDrive) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsDrive) srcFs, ok := src.(*FsDrive)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
@ -752,14 +752,14 @@ func (dstFs *FsDrive) DirMove(src fs.Fs) error {
} }
// Check if destination exists // Check if destination exists
dstFs.dirCache.ResetRoot() f.dirCache.ResetRoot()
err := dstFs.dirCache.FindRoot(false) err := f.dirCache.FindRoot(false)
if err == nil { if err == nil {
return fs.ErrorDirExists return fs.ErrorDirExists
} }
// Find ID of parent // Find ID of parent
leaf, directoryId, err := dstFs.dirCache.FindPath(dstFs.root, true) leaf, directoryID, err := f.dirCache.FindPath(f.root, true)
if err != nil { if err != nil {
return err return err
} }
@ -767,9 +767,9 @@ func (dstFs *FsDrive) DirMove(src fs.Fs) error {
// Do the move // Do the move
patch := drive.File{ patch := drive.File{
Title: leaf, Title: leaf,
Parents: []*drive.ParentReference{{Id: directoryId}}, Parents: []*drive.ParentReference{{Id: directoryID}},
} }
_, err = dstFs.svc.Files.Patch(srcFs.dirCache.RootID(), &patch).Do() _, err = f.svc.Files.Patch(srcFs.dirCache.RootID(), &patch).Do()
if err != nil { if err != nil {
return err return err
} }
@ -779,7 +779,7 @@ func (dstFs *FsDrive) DirMove(src fs.Fs) error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectDrive) Fs() fs.Fs { func (o *FsObjectDrive) Fs() fs.Fs {
return o.drive return o.drive
} }
@ -792,7 +792,7 @@ func (o *FsObjectDrive) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectDrive) Remote() string { func (o *FsObjectDrive) Remote() string {
return o.remote return o.remote
} }
@ -822,12 +822,12 @@ func (o *FsObjectDrive) readMetaData() (err error) {
return nil return nil
} }
leaf, directoryId, err := o.drive.dirCache.FindPath(o.remote, false) leaf, directoryID, err := o.drive.dirCache.FindPath(o.remote, false)
if err != nil { if err != nil {
return err return err
} }
found, err := o.drive.listAll(directoryId, leaf, false, true, func(item *drive.File) bool { found, err := o.drive.listAll(directoryID, leaf, false, true, func(item *drive.File) bool {
if item.Title == leaf { if item.Title == leaf {
o.setMetaData(item) o.setMetaData(item)
return true return true
@ -863,7 +863,7 @@ func (o *FsObjectDrive) ModTime() time.Time {
return modTime return modTime
} }
// Sets the modification time of the drive fs object // SetModTime sets the modification time of the drive fs object
func (o *FsObjectDrive) SetModTime(modTime time.Time) { func (o *FsObjectDrive) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
@ -890,7 +890,7 @@ func (o *FsObjectDrive) SetModTime(modTime time.Time) {
o.setMetaData(info) o.setMetaData(info)
} }
// Is this object storable // Storable returns a boolean as to whether this object is storable
func (o *FsObjectDrive) Storable() bool { func (o *FsObjectDrive) Storable() bool {
return true return true
} }

8
drive/upload.go
View File

@ -52,8 +52,8 @@ type resumableUpload struct {
// Upload the io.Reader in of size bytes with contentType and info // Upload the io.Reader in of size bytes with contentType and info
func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *drive.File, remote string) (*drive.File, error) { func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *drive.File, remote string) (*drive.File, error) {
fileId := info.Id fileID := info.Id
var body io.Reader = nil var body io.Reader
body, err := googleapi.WithoutDataWrapper.JSONReader(info) body, err := googleapi.WithoutDataWrapper.JSONReader(info)
if err != nil { if err != nil {
return nil, err return nil, err
@ -63,7 +63,7 @@ func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *dri
params.Set("uploadType", "resumable") params.Set("uploadType", "resumable")
urls := "https://www.googleapis.com/upload/drive/v2/files" urls := "https://www.googleapis.com/upload/drive/v2/files"
method := "POST" method := "POST"
if fileId != "" { if fileID != "" {
params.Set("setModifiedDate", "true") params.Set("setModifiedDate", "true")
urls += "/{fileId}" urls += "/{fileId}"
method = "PUT" method = "PUT"
@ -71,7 +71,7 @@ func (f *FsDrive) Upload(in io.Reader, size int64, contentType string, info *dri
urls += "?" + params.Encode() urls += "?" + params.Encode()
req, _ := http.NewRequest(method, urls, body) req, _ := http.NewRequest(method, urls, body)
googleapi.Expand(req.URL, map[string]string{ googleapi.Expand(req.URL, map[string]string{
"fileId": fileId, "fileId": fileID,
}) })
req.Header.Set("Content-Type", "application/json; charset=UTF-8") req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("X-Upload-Content-Type", contentType) req.Header.Set("X-Upload-Content-Type", contentType)

40
dropbox/dropbox.go
View File

@ -1,4 +1,4 @@
// Dropbox interface // Package dropbox provides an interface to Dropbox object storage
package dropbox package dropbox
/* /*
@ -44,7 +44,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "dropbox", Name: "dropbox",
NewFs: NewFs, NewFs: NewFs,
Config: configHelper, Config: configHelper,
@ -112,12 +112,12 @@ type FsObjectDropbox struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsDropbox) Name() string { func (f *FsDropbox) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsDropbox) Root() string { func (f *FsDropbox) Root() string {
return f.root return f.root
} }
@ -217,7 +217,7 @@ func (f *FsDropbox) newFsObjectWithInfo(remote string, info *dropbox.Entry) fs.O
return o return o
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsDropbox) NewFsObject(remote string) fs.Object { func (f *FsDropbox) NewFsObject(remote string) fs.Object {
@ -243,7 +243,7 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
// Track path component case, it could be different for entries coming from DropBox API // Track path component case, it could be different for entries coming from DropBox API
// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us // See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
// and https://github.com/ncw/rclone/issues/53 // and https://github.com/ncw/rclone/issues/53
nameTree := NewNameTree() nameTree := newNameTree()
cursor := "" cursor := ""
for { for {
deltaPage, err := f.db.Delta(cursor, f.slashRoot) deltaPage, err := f.db.Delta(cursor, f.slashRoot)
@ -317,7 +317,7 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
nameTree.WalkFiles(f.root, walkFunc) nameTree.WalkFiles(f.root, walkFunc)
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsDropbox) List() fs.ObjectsChan { func (f *FsDropbox) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
go func() { go func() {
@ -327,7 +327,7 @@ func (f *FsDropbox) List() fs.ObjectsChan {
return out return out
} }
// Walk the path returning a channel of FsObjects // ListDir walks the path returning a channel of FsObjects
func (f *FsDropbox) ListDir() fs.DirChan { func (f *FsDropbox) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
@ -412,7 +412,7 @@ func (f *FsDropbox) Rmdir() error {
return f.Purge() return f.Purge()
} }
// Return the precision // Precision returns the precision
func (f *FsDropbox) Precision() time.Duration { func (f *FsDropbox) Precision() time.Duration {
return fs.ModTimeNotSupported return fs.ModTimeNotSupported
} }
@ -466,7 +466,7 @@ func (f *FsDropbox) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (dstFs *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) { func (f *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectDropbox) srcObj, ok := src.(*FsObjectDropbox)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
@ -474,11 +474,11 @@ func (dstFs *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) {
} }
// Temporary FsObject under construction // Temporary FsObject under construction
dstObj := &FsObjectDropbox{dropbox: dstFs, remote: remote} dstObj := &FsObjectDropbox{dropbox: f, remote: remote}
srcPath := srcObj.remotePath() srcPath := srcObj.remotePath()
dstPath := dstObj.remotePath() dstPath := dstObj.remotePath()
entry, err := dstFs.db.Move(srcPath, dstPath) entry, err := f.db.Move(srcPath, dstPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("Move failed: %s", err) return nil, fmt.Errorf("Move failed: %s", err)
} }
@ -486,14 +486,14 @@ func (dstFs *FsDropbox) Move(src fs.Object, remote string) (fs.Object, error) {
return dstObj, nil return dstObj, nil
} }
// Move src to this remote using server side move operations. // DirMove moves src to this remote using server side move operations.
// //
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (dstFs *FsDropbox) DirMove(src fs.Fs) error { func (f *FsDropbox) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsDropbox) srcFs, ok := src.(*FsDropbox)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
@ -501,13 +501,13 @@ func (dstFs *FsDropbox) DirMove(src fs.Fs) error {
} }
// Check if destination exists // Check if destination exists
entry, err := dstFs.db.Metadata(dstFs.slashRoot, false, false, "", "", metadataLimit) entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
if err == nil && !entry.IsDeleted { if err == nil && !entry.IsDeleted {
return fs.ErrorDirExists return fs.ErrorDirExists
} }
// Do the move // Do the move
_, err = dstFs.db.Move(srcFs.slashRoot, dstFs.slashRoot) _, err = f.db.Move(srcFs.slashRoot, f.slashRoot)
if err != nil { if err != nil {
return fmt.Errorf("MoveDir failed: %v", err) return fmt.Errorf("MoveDir failed: %v", err)
} }
@ -516,7 +516,7 @@ func (dstFs *FsDropbox) DirMove(src fs.Fs) error {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectDropbox) Fs() fs.Fs { func (o *FsObjectDropbox) Fs() fs.Fs {
return o.dropbox return o.dropbox
} }
@ -529,7 +529,7 @@ func (o *FsObjectDropbox) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectDropbox) Remote() string { func (o *FsObjectDropbox) Remote() string {
return o.remote return o.remote
} }
@ -618,7 +618,7 @@ func (o *FsObjectDropbox) ModTime() time.Time {
return o.modTime return o.modTime
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
// //
// Commits the datastore // Commits the datastore
func (o *FsObjectDropbox) SetModTime(modTime time.Time) { func (o *FsObjectDropbox) SetModTime(modTime time.Time) {
@ -626,7 +626,7 @@ func (o *FsObjectDropbox) SetModTime(modTime time.Time) {
return return
} }
// Is this object storable // Storable returns whether this object is storable
func (o *FsObjectDropbox) Storable() bool { func (o *FsObjectDropbox) Storable() bool {
return true return true
} }

33
dropbox/nametree.go
View File

@ -9,9 +9,9 @@ import (
"github.com/stacktic/dropbox" "github.com/stacktic/dropbox"
) )
type NameTreeNode struct { type nameTreeNode struct {
// Map from lowercase directory name to tree node // Map from lowercase directory name to tree node
Directories map[string]*NameTreeNode Directories map[string]*nameTreeNode
// Map from file name (case sensitive) to dropbox entry // Map from file name (case sensitive) to dropbox entry
Files map[string]*dropbox.Entry Files map[string]*dropbox.Entry
@ -22,27 +22,26 @@ type NameTreeNode struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
func newNameTreeNode(caseCorrectName string) *NameTreeNode { func newNameTreeNode(caseCorrectName string) *nameTreeNode {
return &NameTreeNode{ return &nameTreeNode{
CaseCorrectName: caseCorrectName, CaseCorrectName: caseCorrectName,
Directories: make(map[string]*NameTreeNode), Directories: make(map[string]*nameTreeNode),
Files: make(map[string]*dropbox.Entry), Files: make(map[string]*dropbox.Entry),
} }
} }
func NewNameTree() *NameTreeNode { func newNameTree() *nameTreeNode {
return newNameTreeNode("") return newNameTreeNode("")
} }
func (tree *NameTreeNode) String() string { func (tree *nameTreeNode) String() string {
if len(tree.CaseCorrectName) == 0 { if len(tree.CaseCorrectName) == 0 {
return "NameTreeNode/<root>" return "nameTreeNode/<root>"
} else {
return fmt.Sprintf("NameTreeNode/%q", tree.CaseCorrectName)
} }
return fmt.Sprintf("nameTreeNode/%q", tree.CaseCorrectName)
} }
func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode { func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode {
if len(path) == 0 { if len(path) == 0 {
// no lookup required, just return root // no lookup required, just return root
return tree return tree
@ -70,7 +69,7 @@ func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode {
return current return current
} }
func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) { func (tree *nameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) {
if len(caseCorrectDirectoryName) == 0 { if len(caseCorrectDirectoryName) == 0 {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath) fs.ErrorLog(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath)
@ -98,7 +97,7 @@ func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCor
} }
} }
func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) { func (tree *nameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) {
node := tree.getTreeNode(parentPath) node := tree.getTreeNode(parentPath)
if node == nil { if node == nil {
return return
@ -113,7 +112,7 @@ func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string,
node.Files[caseCorrectFileName] = dropboxEntry node.Files[caseCorrectFileName] = dropboxEntry
} }
func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string { func (tree *nameTreeNode) GetPathWithCorrectCase(path string) *string {
if path == "" { if path == "" {
empty := "" empty := ""
return &empty return &empty
@ -144,9 +143,9 @@ func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string {
return &resultString return &resultString
} }
type NameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry) type nameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry)
func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFileWalkFunc) { func (tree *nameTreeNode) walkFilesRec(currentPath string, walkFunc nameTreeFileWalkFunc) {
var prefix string var prefix string
if currentPath == "" { if currentPath == "" {
prefix = "" prefix = ""
@ -170,7 +169,7 @@ func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFile
} }
} }
func (tree *NameTreeNode) WalkFiles(rootPath string, walkFunc NameTreeFileWalkFunc) { func (tree *nameTreeNode) WalkFiles(rootPath string, walkFunc nameTreeFileWalkFunc) {
node := tree.getTreeNode(rootPath) node := tree.getTreeNode(rootPath)
if node == nil { if node == nil {
return return

20
dropbox/nametree_test.go
View File

@ -1,10 +1,10 @@
package dropbox_test package dropbox
import ( import (
"github.com/ncw/rclone/dropbox" "testing"
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
dropboxapi "github.com/stacktic/dropbox" dropboxapi "github.com/stacktic/dropbox"
"testing"
) )
func assert(t *testing.T, shouldBeTrue bool, failMessage string) { func assert(t *testing.T, shouldBeTrue bool, failMessage string) {
@ -16,7 +16,7 @@ func assert(t *testing.T, shouldBeTrue bool, failMessage string) {
func TestPutCaseCorrectDirectoryName(t *testing.T) { func TestPutCaseCorrectDirectoryName(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutCaseCorrectDirectoryName("a/b", "C") tree.PutCaseCorrectDirectoryName("a/b", "C")
assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty") assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty")
@ -36,7 +36,7 @@ func TestPutCaseCorrectDirectoryName(t *testing.T) {
func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) { func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutCaseCorrectDirectoryName("/a", "C") tree.PutCaseCorrectDirectoryName("/a", "C")
tree.PutCaseCorrectDirectoryName("b/", "C") tree.PutCaseCorrectDirectoryName("b/", "C")
tree.PutCaseCorrectDirectoryName("a//b", "C") tree.PutCaseCorrectDirectoryName("a//b", "C")
@ -47,7 +47,7 @@ func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) { func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutCaseCorrectDirectoryName("", "C") tree.PutCaseCorrectDirectoryName("", "C")
c := tree.Directories["c"] c := tree.Directories["c"]
@ -59,7 +59,7 @@ func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
func TestGetPathWithCorrectCase(t *testing.T) { func TestGetPathWithCorrectCase(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutCaseCorrectDirectoryName("a", "C") tree.PutCaseCorrectDirectoryName("a", "C")
assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available") assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
@ -72,7 +72,7 @@ func TestGetPathWithCorrectCase(t *testing.T) {
func TestPutAndWalk(t *testing.T) { func TestPutAndWalk(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
tree.PutCaseCorrectDirectoryName("", "A") tree.PutCaseCorrectDirectoryName("", "A")
@ -92,7 +92,7 @@ func TestPutAndWalk(t *testing.T) {
func TestPutAndWalkWithPrefix(t *testing.T) { func TestPutAndWalkWithPrefix(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
tree.PutCaseCorrectDirectoryName("", "A") tree.PutCaseCorrectDirectoryName("", "A")
@ -112,7 +112,7 @@ func TestPutAndWalkWithPrefix(t *testing.T) {
func TestPutAndWalkIncompleteTree(t *testing.T) { func TestPutAndWalkIncompleteTree(t *testing.T) {
errors := fs.Stats.GetErrors() errors := fs.Stats.GetErrors()
tree := dropbox.NewNameTree() tree := newNameTree()
tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"}) tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) { walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {

20
fs/accounting.go
View File

@ -70,7 +70,7 @@ func (ip *inProgress) get(name string) *Account {
// Strings returns all the strings in the stringSet // Strings returns all the strings in the stringSet
func (ss stringSet) Strings() []string { func (ss stringSet) Strings() []string {
strings := make([]string, 0, len(ss)) strings := make([]string, 0, len(ss))
for name, _ := range ss { for name := range ss {
var out string var out string
if acc := Stats.inProgress.get(name); acc != nil { if acc := Stats.inProgress.get(name); acc != nil {
out = acc.String() out = acc.String()
@ -89,7 +89,7 @@ func (ss stringSet) String() string {
return strings.Join(ss.Strings(), "\n") return strings.Join(ss.Strings(), "\n")
} }
// Stats limits and accounts all transfers // StatsInfo limits and accounts all transfers
type StatsInfo struct { type StatsInfo struct {
lock sync.RWMutex lock sync.RWMutex
bytes int64 bytes int64
@ -117,12 +117,12 @@ func (s *StatsInfo) String() string {
s.lock.RLock() s.lock.RLock()
defer s.lock.RUnlock() defer s.lock.RUnlock()
dt := time.Now().Sub(s.start) dt := time.Now().Sub(s.start)
dt_seconds := dt.Seconds() dtSeconds := dt.Seconds()
speed := 0.0 speed := 0.0
if dt > 0 { if dt > 0 {
speed = float64(s.bytes) / 1024 / dt_seconds speed = float64(s.bytes) / 1024 / dtSeconds
} }
dt_rounded := dt - (dt % (time.Second / 10)) dtRounded := dt - (dt % (time.Second / 10))
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
fmt.Fprintf(buf, ` fmt.Fprintf(buf, `
Transferred: %10d Bytes (%7.2f kByte/s) Transferred: %10d Bytes (%7.2f kByte/s)
@ -135,7 +135,7 @@ Elapsed time: %10v
s.errors, s.errors,
s.checks, s.checks,
s.transfers, s.transfers,
dt_rounded) dtRounded)
if len(s.checking) > 0 { if len(s.checking) > 0 {
fmt.Fprintf(buf, "Checking:\n%s\n", s.checking) fmt.Fprintf(buf, "Checking:\n%s\n", s.checking)
} }
@ -199,7 +199,7 @@ func (s *StatsInfo) Errored() bool {
func (s *StatsInfo) Error() { func (s *StatsInfo) Error() {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
s.errors += 1 s.errors++
} }
// Checking adds a check into the stats // Checking adds a check into the stats
@ -214,7 +214,7 @@ func (s *StatsInfo) DoneChecking(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
delete(s.checking, o.Remote()) delete(s.checking, o.Remote())
s.checks += 1 s.checks++
} }
// GetTransfers reads the number of transfers // GetTransfers reads the number of transfers
@ -236,7 +236,7 @@ func (s *StatsInfo) DoneTransferring(o Object) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()
delete(s.transferring, o.Remote()) delete(s.transferring, o.Remote())
s.transfers += 1 s.transfers++
} }
// Account limits and accounts for one transfer // Account limits and accounts for one transfer
@ -324,7 +324,7 @@ func (file *Account) Read(p []byte) (n int, err error) {
return return
} }
// Returns bytes read as well as the size. // Progress returns bytes read as well as the size.
// Size can be <= 0 if the size is unknown. // Size can be <= 0 if the size is unknown.
func (file *Account) Progress() (bytes, size int64) { func (file *Account) Progress() (bytes, size int64) {
if file == nil { if file == nil {

39
fs/config.go
View File

@ -26,17 +26,18 @@ const (
configFileName = ".rclone.conf" configFileName = ".rclone.conf"
) )
// SizeSuffix is parsed by flag with k/M/G suffixes
type SizeSuffix int64 type SizeSuffix int64
// Global // Global
var ( var (
// Config file // ConfigFile is the config file data structure
ConfigFile *goconfig.ConfigFile ConfigFile *goconfig.ConfigFile
// Home directory // HomeDir is the home directory of the user
HomeDir = configHome() HomeDir = configHome()
// Config file path // ConfigPath points to the config file
ConfigPath = path.Join(HomeDir, configFileName) ConfigPath = path.Join(HomeDir, configFileName)
// Global config // Config is the global config
Config = &ConfigInfo{} Config = &ConfigInfo{}
// Flags // Flags
verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff") verbose = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
@ -145,7 +146,7 @@ func Reveal(y string) string {
return string(x) return string(x)
} }
// Filesystem config options // ConfigInfo is filesystem config options
type ConfigInfo struct { type ConfigInfo struct {
Verbose bool Verbose bool
Quiet bool Quiet bool
@ -192,7 +193,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper {
return t return t
} }
// Transport returns an http.Client with the correct timeouts // Client returns an http.Client with the correct timeouts
func (ci *ConfigInfo) Client() *http.Client { func (ci *ConfigInfo) Client() *http.Client {
return &http.Client{ return &http.Client{
Transport: ci.Transport(), Transport: ci.Transport(),
@ -220,7 +221,7 @@ func configHome() string {
return "" return ""
} }
// Loads the config file // LoadConfig loads the config file
func LoadConfig() { func LoadConfig() {
// Read some flags if set // Read some flags if set
// //
@ -255,7 +256,7 @@ func LoadConfig() {
startTokenBucket() startTokenBucket()
} }
// Save configuration file. // SaveConfig saves configuration file.
func SaveConfig() { func SaveConfig() {
err := goconfig.SaveConfigFile(ConfigFile, ConfigPath) err := goconfig.SaveConfigFile(ConfigFile, ConfigPath)
if err != nil { if err != nil {
@ -267,7 +268,7 @@ func SaveConfig() {
} }
} }
// Show an overview of the config file // ShowRemotes shows an overview of the config file
func ShowRemotes() { func ShowRemotes() {
remotes := ConfigFile.GetSectionList() remotes := ConfigFile.GetSectionList()
if len(remotes) == 0 { if len(remotes) == 0 {
@ -288,7 +289,7 @@ func ChooseRemote() string {
return Choose("remote", remotes, nil, false) return Choose("remote", remotes, nil, false)
} }
// Read some input // ReadLine reads some input
func ReadLine() string { func ReadLine() string {
buf := bufio.NewReader(os.Stdin) buf := bufio.NewReader(os.Stdin)
line, err := buf.ReadString('\n') line, err := buf.ReadString('\n')
@ -320,7 +321,7 @@ func Command(commands []string) byte {
} }
} }
// Asks the user for Yes or No and returns true or false // Confirm asks the user for Yes or No and returns true or false
func Confirm() bool { func Confirm() bool {
return Command([]string{"yYes", "nNo"}) == 'y' return Command([]string{"yYes", "nNo"}) == 'y'
} }
@ -357,7 +358,7 @@ func Choose(what string, defaults, help []string, newOk bool) string {
} }
} }
// Show the contents of the remote // ShowRemote shows the contents of the remote
func ShowRemote(name string) { func ShowRemote(name string) {
fmt.Printf("--------------------\n") fmt.Printf("--------------------\n")
fmt.Printf("[%s]\n", name) fmt.Printf("[%s]\n", name)
@ -367,7 +368,7 @@ func ShowRemote(name string) {
fmt.Printf("--------------------\n") fmt.Printf("--------------------\n")
} }
// Print the contents of the remote and ask if it is OK // OkRemote prints the contents of the remote and ask if it is OK
func OkRemote(name string) bool { func OkRemote(name string) bool {
ShowRemote(name) ShowRemote(name)
switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i { switch i := Command([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}); i {
@ -384,7 +385,7 @@ func OkRemote(name string) bool {
return false return false
} }
// Runs the config helper for the remote if needed // RemoteConfig runs the config helper for the remote if needed
func RemoteConfig(name string) { func RemoteConfig(name string) {
fmt.Printf("Remote config\n") fmt.Printf("Remote config\n")
fsName := ConfigFile.MustValue(name, "type") fsName := ConfigFile.MustValue(name, "type")
@ -400,7 +401,7 @@ func RemoteConfig(name string) {
} }
} }
// Choose an option // ChooseOption asks the user to choose an option
func ChooseOption(o *Option) string { func ChooseOption(o *Option) string {
fmt.Println(o.Help) fmt.Println(o.Help)
if len(o.Examples) > 0 { if len(o.Examples) > 0 {
@ -416,7 +417,7 @@ func ChooseOption(o *Option) string {
return ReadLine() return ReadLine()
} }
// Make a new remote // NewRemote make a new remote from its name
func NewRemote(name string) { func NewRemote(name string) {
fmt.Printf("What type of source is it?\n") fmt.Printf("What type of source is it?\n")
types := []string{} types := []string{}
@ -440,7 +441,7 @@ func NewRemote(name string) {
EditRemote(name) EditRemote(name)
} }
// Edit a remote // EditRemote gets the user to edit a remote
func EditRemote(name string) { func EditRemote(name string) {
ShowRemote(name) ShowRemote(name)
fmt.Printf("Edit remote\n") fmt.Printf("Edit remote\n")
@ -462,13 +463,13 @@ func EditRemote(name string) {
SaveConfig() SaveConfig()
} }
// Delete a remote // DeleteRemote gets the user to delete a remote
func DeleteRemote(name string) { func DeleteRemote(name string) {
ConfigFile.DeleteSection(name) ConfigFile.DeleteSection(name)
SaveConfig() SaveConfig()
} }
// Edit the config file interactively // EditConfig edits the config file interactively
func EditConfig() { func EditConfig() {
for { for {
haveRemotes := len(ConfigFile.GetSectionList()) != 0 haveRemotes := len(ConfigFile.GetSectionList()) != 0

93
fs/fs.go
View File

@ -1,4 +1,4 @@
// Generic file system interface for rclone object storage systems // Package fs is a generic file system interface for rclone object storage systems
package fs package fs
import ( import (
@ -12,26 +12,27 @@ import (
// Constants // Constants
const ( const (
// User agent for Fs which can set it // UserAgent for Fs which can set it
UserAgent = "rclone/" + Version UserAgent = "rclone/" + Version
// Very large precision value to show mod time isn't supported // ModTimeNotSupported is a very large precision value to show
// mod time isn't supported on this Fs
ModTimeNotSupported = 100 * 365 * 24 * time.Hour ModTimeNotSupported = 100 * 365 * 24 * time.Hour
) )
// Globals // Globals
var ( var (
// Filesystem registry // Filesystem registry
fsRegistry []*FsInfo fsRegistry []*Info
// Error returned by NewFs if not found in config file // ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file") ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes") ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes")
ErrorCantMove = fmt.Errorf("Can't copy object - incompatible remotes") ErrorCantMove = fmt.Errorf("Can't copy object - incompatible remotes")
ErrorCantDirMove = fmt.Errorf("Can't copy directory - incompatible remotes") ErrorCantDirMove = fmt.Errorf("Can't copy directory - incompatible remotes")
ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists") ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists")
) )
// Filesystem info // Info information about a filesystem
type FsInfo struct { type Info struct {
// Name of this fs // Name of this fs
Name string Name string
// Create a new file system. If root refers to an existing // Create a new file system. If root refers to an existing
@ -44,7 +45,7 @@ type FsInfo struct {
Options []Option Options []Option
} }
// An options for a Fs // Option is describes an option for the config wizard
type Option struct { type Option struct {
Name string Name string
Help string Help string
@ -52,7 +53,7 @@ type Option struct {
Examples []OptionExample Examples []OptionExample
} }
// An example for an option // OptionExample describes an example for an Option
type OptionExample struct { type OptionExample struct {
Value string Value string
Help string Help string
@ -61,16 +62,16 @@ type OptionExample struct {
// Register a filesystem // Register a filesystem
// //
// Fs modules should use this in an init() function // Fs modules should use this in an init() function
func Register(info *FsInfo) { func Register(info *Info) {
fsRegistry = append(fsRegistry, info) fsRegistry = append(fsRegistry, info)
} }
// A Filesystem, describes the local filesystem and the remote object store // Fs is the interface a cloud storage system must provide
type Fs interface { type Fs interface {
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
Name() string Name() string
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
Root() string Root() string
// String returns a description of the FS // String returns a description of the FS
@ -79,10 +80,10 @@ type Fs interface {
// List the Fs into a channel // List the Fs into a channel
List() ObjectsChan List() ObjectsChan
// List the Fs directories/buckets/containers into a channel // ListDir lists the Fs directories/buckets/containers into a channel
ListDir() DirChan ListDir() DirChan
// Find the Object at remote. Returns nil if can't be found // NewFsObject finds the Object at remote. Returns nil if can't be found
NewFsObject(remote string) Object NewFsObject(remote string) Object
// Put in to the remote path with the modTime given of the given size // Put in to the remote path with the modTime given of the given size
@ -92,12 +93,12 @@ type Fs interface {
// nil and the error // nil and the error
Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error)
// Make the directory (container, bucket) // Mkdir makes the directory (container, bucket)
// //
// Shouldn't return an error if it already exists // Shouldn't return an error if it already exists
Mkdir() error Mkdir() error
// Remove the directory (container, bucket) if empty // Rmdir removes the directory (container, bucket) if empty
// //
// Return an error if it doesn't exist or isn't empty // Return an error if it doesn't exist or isn't empty
Rmdir() error Rmdir() error
@ -106,8 +107,7 @@ type Fs interface {
Precision() time.Duration Precision() time.Duration
} }
// A filesystem like object which can either be a remote object or a // Object is a filesystem like object provided by an Fs
// local file/directory
type Object interface { type Object interface {
// String returns a description of the Object // String returns a description of the Object
String() string String() string
@ -145,7 +145,7 @@ type Object interface {
Remove() error Remove() error
} }
// Optional interfaces // Purger is an optional interfaces for Fs
type Purger interface { type Purger interface {
// Purge all files in the root and the root directory // Purge all files in the root and the root directory
// //
@ -156,6 +156,7 @@ type Purger interface {
Purge() error Purge() error
} }
// Copier is an optional interface for Fs
type Copier interface { type Copier interface {
// Copy src to this remote using server side copy operations. // Copy src to this remote using server side copy operations.
// //
@ -169,6 +170,7 @@ type Copier interface {
Copy(src Object, remote string) (Object, error) Copy(src Object, remote string) (Object, error)
} }
// Mover is an optional interface for Fs
type Mover interface { type Mover interface {
// Move src to this remote using server side move operations. // Move src to this remote using server side move operations.
// //
@ -182,8 +184,10 @@ type Mover interface {
Move(src Object, remote string) (Object, error) Move(src Object, remote string) (Object, error)
} }
// DirMover is an optional interface for Fs
type DirMover interface { type DirMover interface {
// Move src to this remote using server side move operations. // DirMove moves src to this remote using server side move
// operations.
// //
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
@ -193,7 +197,8 @@ type DirMover interface {
DirMove(src Fs) error DirMove(src Fs) error
} }
// An optional interface for error as to whether the operation should be retried // Retry is optional interface for error as to whether the operation
// should be retried at a high level.
// //
// This should be returned from Update or Put methods as required // This should be returned from Update or Put methods as required
type Retry interface { type Retry interface {
@ -201,7 +206,7 @@ type Retry interface {
Retry() bool Retry() bool
} }
// A type of error // retryError is a type of error
type retryError string type retryError string
// Error interface // Error interface
@ -228,7 +233,7 @@ type plainRetryError struct {
} }
// Retry interface // Retry interface
func (_ plainRetryError) Retry() bool { func (err plainRetryError) Retry() bool {
return true return true
} }
@ -240,21 +245,22 @@ func RetryError(err error) error {
return plainRetryError{err} return plainRetryError{err}
} }
// A channel of Objects // ObjectsChan is a channel of Objects
type ObjectsChan chan Object type ObjectsChan chan Object
// A slice of Objects // Objects is a slice of Object~s
type Objects []Object type Objects []Object
// A pair of Objects // ObjectPair is a pair of Objects used to describe a potential copy
// operation.
type ObjectPair struct { type ObjectPair struct {
src, dst Object src, dst Object
} }
// A channel of ObjectPair // ObjectPairChan is a channel of ObjectPair
type ObjectPairChan chan ObjectPair type ObjectPairChan chan ObjectPair
// A structure of directory/container/bucket lists // Dir describes a directory for directory/container/bucket lists
type Dir struct { type Dir struct {
Name string // name of the directory Name string // name of the directory
When time.Time // modification or creation time - IsZero for unknown When time.Time // modification or creation time - IsZero for unknown
@ -262,13 +268,13 @@ type Dir struct {
Count int64 // number of objects -1 for unknown Count int64 // number of objects -1 for unknown
} }
// A channel of Dir objects // DirChan is a channel of Dir objects
type DirChan chan *Dir type DirChan chan *Dir
// Finds a FsInfo object for the name passed in // Find looks for an Info object for the name passed in
// //
// Services are looked up in the config file // Services are looked up in the config file
func Find(name string) (*FsInfo, error) { func Find(name string) (*Info, error) {
for _, item := range fsRegistry { for _, item := range fsRegistry {
if item.Name == name { if item.Name == name {
return item, nil return item, nil
@ -297,7 +303,7 @@ func NewFs(path string) (Fs, error) {
var err error var err error
fsName, err = ConfigFile.GetValue(configName, "type") fsName, err = ConfigFile.GetValue(configName, "type")
if err != nil { if err != nil {
return nil, NotFoundInConfigFile return nil, ErrorNotFoundInConfigFile
} }
} }
fs, err := Find(fsName) fs, err := Find(fsName)
@ -309,7 +315,7 @@ func NewFs(path string) (Fs, error) {
return fs.NewFs(configName, fsPath) return fs.NewFs(configName, fsPath)
} }
// Outputs log for object // OutputLog logs for an object
func OutputLog(o interface{}, text string, args ...interface{}) { func OutputLog(o interface{}, text string, args ...interface{}) {
description := "" description := ""
if o != nil { if o != nil {
@ -319,22 +325,23 @@ func OutputLog(o interface{}, text string, args ...interface{}) {
log.Print(description + out) log.Print(description + out)
} }
// Write debuging output for this Object or Fs // Debug writes debuging output for this Object or Fs
func Debug(o interface{}, text string, args ...interface{}) { func Debug(o interface{}, text string, args ...interface{}) {
if Config.Verbose { if Config.Verbose {
OutputLog(o, text, args...) OutputLog(o, text, args...)
} }
} }
// Write log output for this Object or Fs // Log writes log output for this Object or Fs
func Log(o interface{}, text string, args ...interface{}) { func Log(o interface{}, text string, args ...interface{}) {
if !Config.Quiet { if !Config.Quiet {
OutputLog(o, text, args...) OutputLog(o, text, args...)
} }
} }
// Write error log output for this Object or Fs // ErrorLog writes error log output for this Object or Fs. It
// Unconditionally logs a message regardless of Config.Quiet or Config.Verbose // unconditionally logs a message regardless of Config.Quiet or
// Config.Verbose.
func ErrorLog(o interface{}, text string, args ...interface{}) { func ErrorLog(o interface{}, text string, args ...interface{}) {
OutputLog(o, text, args...) OutputLog(o, text, args...)
} }

15
fs/limited.go
View File

@ -6,7 +6,8 @@ import (
"time" "time"
) )
// This defines a Limited Fs which can only return the Objects passed in from the Fs passed in // Limited defines a Fs which can only return the Objects passed in
// from the Fs passed in
type Limited struct { type Limited struct {
objects []Object objects []Object
fs Fs fs Fs
@ -21,12 +22,12 @@ func NewLimited(fs Fs, objects ...Object) Fs {
return f return f
} }
// The name of the remote (as passed into NewFs) // Name is name of the remote (as passed into NewFs)
func (f *Limited) Name() string { func (f *Limited) Name() string {
return f.fs.Name() // return name of underlying remote return f.fs.Name() // return name of underlying remote
} }
// The root of the remote (as passed into NewFs) // Root is the root of the remote (as passed into NewFs)
func (f *Limited) Root() string { func (f *Limited) Root() string {
return f.fs.Root() // return root of underlying remote return f.fs.Root() // return root of underlying remote
} }
@ -48,14 +49,14 @@ func (f *Limited) List() ObjectsChan {
return out return out
} }
// List the Fs directories/buckets/containers into a channel // ListDir lists the Fs directories/buckets/containers into a channel
func (f *Limited) ListDir() DirChan { func (f *Limited) ListDir() DirChan {
out := make(DirChan, Config.Checkers) out := make(DirChan, Config.Checkers)
close(out) close(out)
return out return out
} }
// Find the Object at remote. Returns nil if can't be found // NewFsObject finds the Object at remote. Returns nil if can't be found
func (f *Limited) NewFsObject(remote string) Object { func (f *Limited) NewFsObject(remote string) Object {
for _, obj := range f.objects { for _, obj := range f.objects {
if obj.Remote() == remote { if obj.Remote() == remote {
@ -78,13 +79,13 @@ func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64
return obj, obj.Update(in, modTime, size) return obj, obj.Update(in, modTime, size)
} }
// Make the directory (container, bucket) // Mkdir make the directory (container, bucket)
func (f *Limited) Mkdir() error { func (f *Limited) Mkdir() error {
// All directories are already made - just ignore // All directories are already made - just ignore
return nil return nil
} }
// Remove the directory (container, bucket) if empty // Rmdir removes the directory (container, bucket) if empty
func (f *Limited) Rmdir() error { func (f *Limited) Rmdir() error {
// Ignore this in a limited fs // Ignore this in a limited fs
return nil return nil

12
fs/loghttp.go
View File

@ -10,16 +10,16 @@ import (
const separator = "------------------------------------------------------------" const separator = "------------------------------------------------------------"
// An http transport which logs the traffic // LoggedTransport is an http transport which logs the traffic
type loggedTransport struct { type LoggedTransport struct {
wrapped http.RoundTripper wrapped http.RoundTripper
logBody bool logBody bool
} }
// NewLoggedTransport wraps the transport passed in and logs all roundtrips // NewLoggedTransport wraps the transport passed in and logs all roundtrips
// including the body if logBody is set. // including the body if logBody is set.
func NewLoggedTransport(transport http.RoundTripper, logBody bool) *loggedTransport { func NewLoggedTransport(transport http.RoundTripper, logBody bool) *LoggedTransport {
return &loggedTransport{ return &LoggedTransport{
wrapped: transport, wrapped: transport,
logBody: logBody, logBody: logBody,
} }
@ -28,7 +28,7 @@ func NewLoggedTransport(transport http.RoundTripper, logBody bool) *loggedTransp
// CancelRequest cancels an in-flight request by closing its // CancelRequest cancels an in-flight request by closing its
// connection. CancelRequest should only be called after RoundTrip has // connection. CancelRequest should only be called after RoundTrip has
// returned. // returned.
func (t *loggedTransport) CancelRequest(req *http.Request) { func (t *LoggedTransport) CancelRequest(req *http.Request) {
if wrapped, ok := t.wrapped.(interface { if wrapped, ok := t.wrapped.(interface {
CancelRequest(*http.Request) CancelRequest(*http.Request)
}); ok { }); ok {
@ -38,7 +38,7 @@ func (t *loggedTransport) CancelRequest(req *http.Request) {
} }
// RoundTrip implements the RoundTripper interface. // RoundTrip implements the RoundTripper interface.
func (t *loggedTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) { func (t *LoggedTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
buf, _ := httputil.DumpRequest(req, t.logBody) buf, _ := httputil.DumpRequest(req, t.logBody)
log.Println(separator) log.Println(separator)
log.Println("HTTP REQUEST") log.Println("HTTP REQUEST")

70
fs/operations.go
View File

@ -11,7 +11,8 @@ import (
"time" "time"
) )
// Work out modify window for fses passed in - sets Config.ModifyWindow // CalculateModifyWindow works out modify window for Fses passed in -
// sets Config.ModifyWindow
// //
// This is the largest modify window of all the fses in use, and the // This is the largest modify window of all the fses in use, and the
// user configured value // user configured value
@ -39,7 +40,7 @@ func Md5sumsEqual(src, dst string) bool {
return src == dst return src == dst
} }
// Check the two files to see if the MD5sums are the same // CheckMd5sums checks the two files to see if the MD5sums are the same
// //
// Returns two bools, the first of which is equality and the second of // Returns two bools, the first of which is equality and the second of
// which is true if either of the MD5SUMs were unset. // which is true if either of the MD5SUMs were unset.
@ -71,7 +72,7 @@ func CheckMd5sums(src, dst Object) (equal bool, unset bool, err error) {
return Md5sumsEqual(srcMd5, dstMd5), false, nil return Md5sumsEqual(srcMd5, dstMd5), false, nil
} }
// Checks to see if the src and dst objects are equal by looking at // Equal checks to see if the src and dst objects are equal by looking at
// size, mtime and MD5SUM // size, mtime and MD5SUM
// //
// If the src and dst size are different then it is considered to be // If the src and dst size are different then it is considered to be
@ -139,7 +140,7 @@ func Equal(src, dst Object) bool {
return true return true
} }
// Returns a guess at the mime type from the extension // MimeType returns a guess at the mime type from the extension
func MimeType(o Object) string { func MimeType(o Object) string {
mimeType := mime.TypeByExtension(path.Ext(o.Remote())) mimeType := mime.TypeByExtension(path.Ext(o.Remote()))
if mimeType == "" { if mimeType == "" {
@ -281,7 +282,7 @@ func checkOne(pair ObjectPair, out ObjectPairChan) {
out <- pair out <- pair
} }
// Read Objects~s on in send to out if they need uploading // PairChecker reads Objects~s on in send to out if they need transferring.
// //
// FIXME potentially doing lots of MD5SUMS at once // FIXME potentially doing lots of MD5SUMS at once
func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup) { func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup) {
@ -294,7 +295,7 @@ func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup) {
} }
} }
// Read Objects on in and copy them // PairCopier reads Objects on in and copies them.
func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) { func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
for pair := range in { for pair := range in {
@ -309,7 +310,8 @@ func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
} }
} }
// Read Objects on in and move them if possible, or copy them if not // PairMover reads Objects on in and moves them if possible, or copies
// them if not
func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) { func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
// See if we have Move available // See if we have Move available
@ -343,14 +345,14 @@ func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
} }
} }
// Delete all the files passed in the channel // DeleteFiles removes all the files passed in the channel
func DeleteFiles(to_be_deleted ObjectsChan) { func DeleteFiles(toBeDeleted ObjectsChan) {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(Config.Transfers) wg.Add(Config.Transfers)
for i := 0; i < Config.Transfers; i++ { for i := 0; i < Config.Transfers; i++ {
go func() { go func() {
defer wg.Done() defer wg.Done()
for dst := range to_be_deleted { for dst := range toBeDeleted {
if Config.DryRun { if Config.DryRun {
Debug(dst, "Not deleting as --dry-run") Debug(dst, "Not deleting as --dry-run")
} else { } else {
@ -385,8 +387,8 @@ func readFilesMap(fs Fs) map[string]Object {
return files return files
} }
// Returns true if fdst and fsrc point to the same underlying Fs // Same returns true if fdst and fsrc point to the same underlying Fs
func FsSame(fdst, fsrc Fs) bool { func Same(fdst, fsrc Fs) bool {
return fdst.Name() == fsrc.Name() && fdst.Root() == fsrc.Root() return fdst.Name() == fsrc.Name() && fdst.Root() == fsrc.Root()
} }
@ -396,7 +398,7 @@ func FsSame(fdst, fsrc Fs) bool {
// //
// If DoMove is true then files will be moved instead of copied // If DoMove is true then files will be moved instead of copied
func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
if FsSame(fdst, fsrc) { if Same(fdst, fsrc) {
ErrorLog(fdst, "Nothing to do as source and destination are the same") ErrorLog(fdst, "Nothing to do as source and destination are the same")
return nil return nil
} }
@ -414,22 +416,22 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
delFiles := readFilesMap(fdst) delFiles := readFilesMap(fdst)
// Read source files checking them off against dest files // Read source files checking them off against dest files
to_be_checked := make(ObjectPairChan, Config.Transfers) toBeChecked := make(ObjectPairChan, Config.Transfers)
to_be_uploaded := make(ObjectPairChan, Config.Transfers) toBeUploaded := make(ObjectPairChan, Config.Transfers)
var checkerWg sync.WaitGroup var checkerWg sync.WaitGroup
checkerWg.Add(Config.Checkers) checkerWg.Add(Config.Checkers)
for i := 0; i < Config.Checkers; i++ { for i := 0; i < Config.Checkers; i++ {
go PairChecker(to_be_checked, to_be_uploaded, &checkerWg) go PairChecker(toBeChecked, toBeUploaded, &checkerWg)
} }
var copierWg sync.WaitGroup var copierWg sync.WaitGroup
copierWg.Add(Config.Transfers) copierWg.Add(Config.Transfers)
for i := 0; i < Config.Transfers; i++ { for i := 0; i < Config.Transfers; i++ {
if DoMove { if DoMove {
go PairMover(to_be_uploaded, fdst, &copierWg) go PairMover(toBeUploaded, fdst, &copierWg)
} else { } else {
go PairCopier(to_be_uploaded, fdst, &copierWg) go PairCopier(toBeUploaded, fdst, &copierWg)
} }
} }
@ -439,18 +441,18 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
dst, found := delFiles[remote] dst, found := delFiles[remote]
if found { if found {
delete(delFiles, remote) delete(delFiles, remote)
to_be_checked <- ObjectPair{src, dst} toBeChecked <- ObjectPair{src, dst}
} else { } else {
// No need to check since doesn't exist // No need to check since doesn't exist
to_be_uploaded <- ObjectPair{src, nil} toBeUploaded <- ObjectPair{src, nil}
} }
} }
close(to_be_checked) close(toBeChecked)
}() }()
Log(fdst, "Waiting for checks to finish") Log(fdst, "Waiting for checks to finish")
checkerWg.Wait() checkerWg.Wait()
close(to_be_uploaded) close(toBeUploaded)
Log(fdst, "Waiting for transfers to finish") Log(fdst, "Waiting for transfers to finish")
copierWg.Wait() copierWg.Wait()
@ -474,19 +476,19 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
return nil return nil
} }
// Syncs fsrc into fdst // Sync fsrc into fdst
func Sync(fdst, fsrc Fs) error { func Sync(fdst, fsrc Fs) error {
return syncCopyMove(fdst, fsrc, true, false) return syncCopyMove(fdst, fsrc, true, false)
} }
// Copies fsrc into fdst // CopyDir copies fsrc into fdst
func CopyDir(fdst, fsrc Fs) error { func CopyDir(fdst, fsrc Fs) error {
return syncCopyMove(fdst, fsrc, false, false) return syncCopyMove(fdst, fsrc, false, false)
} }
// Moves fsrc into fdst // MoveDir moves fsrc into fdst
func MoveDir(fdst, fsrc Fs) error { func MoveDir(fdst, fsrc Fs) error {
if FsSame(fdst, fsrc) { if Same(fdst, fsrc) {
ErrorLog(fdst, "Nothing to do as source and destination are the same") ErrorLog(fdst, "Nothing to do as source and destination are the same")
return nil return nil
} }
@ -517,7 +519,7 @@ func MoveDir(fdst, fsrc Fs) error {
return Purge(fsrc) return Purge(fsrc)
} }
// Checks the files in fsrc and fdst according to Size and MD5SUM // Check the files in fsrc and fdst according to Size and MD5SUM
func Check(fdst, fsrc Fs) error { func Check(fdst, fsrc Fs) error {
Log(fdst, "Building file list") Log(fdst, "Building file list")
@ -597,7 +599,7 @@ func Check(fdst, fsrc Fs) error {
return nil return nil
} }
// List the Fs to the supplied function // ListFn lists the Fs to the supplied function
// //
// Lists in parallel which may get them out of order // Lists in parallel which may get them out of order
func ListFn(f Fs, fn func(Object)) error { func ListFn(f Fs, fn func(Object)) error {
@ -639,7 +641,7 @@ func List(f Fs, w io.Writer) error {
}) })
} }
// List the Fs to the supplied writer // ListLong lists the Fs to the supplied writer
// //
// Shows size, mod time and path // Shows size, mod time and path
// //
@ -653,7 +655,7 @@ func ListLong(f Fs, w io.Writer) error {
}) })
} }
// List the Fs to the supplied writer // Md5sum list the Fs to the supplied writer
// //
// Produces the same output as the md5sum command // Produces the same output as the md5sum command
// //
@ -671,7 +673,7 @@ func Md5sum(f Fs, w io.Writer) error {
}) })
} }
// List the directories/buckets/containers in the Fs to the supplied writer // ListDir lists the directories/buckets/containers in the Fs to the supplied writer
func ListDir(f Fs, w io.Writer) error { func ListDir(f Fs, w io.Writer) error {
for dir := range f.ListDir() { for dir := range f.ListDir() {
syncFprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name) syncFprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
@ -679,7 +681,7 @@ func ListDir(f Fs, w io.Writer) error {
return nil return nil
} }
// Makes a destination directory or container // Mkdir makes a destination directory or container
func Mkdir(f Fs) error { func Mkdir(f Fs) error {
err := f.Mkdir() err := f.Mkdir()
if err != nil { if err != nil {
@ -689,7 +691,7 @@ func Mkdir(f Fs) error {
return nil return nil
} }
// Removes a container but not if not empty // Rmdir removes a container but not if not empty
func Rmdir(f Fs) error { func Rmdir(f Fs) error {
if Config.DryRun { if Config.DryRun {
Log(f, "Not deleting as dry run is set") Log(f, "Not deleting as dry run is set")
@ -703,7 +705,7 @@ func Rmdir(f Fs) error {
return nil return nil
} }
// Removes a container and all of its contents // Purge removes a container and all of its contents
// //
// FIXME doesn't delete local directories // FIXME doesn't delete local directories
func Purge(f Fs) error { func Purge(f Fs) error {

1
fs/version.go
View File

@ -1,3 +1,4 @@
package fs package fs
// Version of rclone
const Version = "v1.20" const Version = "v1.20"

36
fstest/fstest.go
View File

@ -1,4 +1,4 @@
// Utilities for testing the Fs // Package fstest provides utilities for testing the Fs
package fstest package fstest
// FIXME put name of test FS in Fs structure // FIXME put name of test FS in Fs structure
@ -22,7 +22,7 @@ func init() {
} }
// Represents an item for checking // Item represents an item for checking
type Item struct { type Item struct {
Path string Path string
Md5sum string Md5sum string
@ -31,7 +31,8 @@ type Item struct {
WinPath string WinPath string
} }
// Checks the times are equal within the precision, returns the delta and a flag // CheckTimeEqualWithPrecision checks the times are equal within the
// precision, returns the delta and a flag
func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) { func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
dt := t0.Sub(t1) dt := t0.Sub(t1)
if dt >= precision || dt <= -precision { if dt >= precision || dt <= -precision {
@ -40,7 +41,7 @@ func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (tim
return dt, true return dt, true
} }
// check the mod time to the given precision // CheckModTime checks the mod time to the given precision
func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) { func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
dt, ok := CheckTimeEqualWithPrecision(modTime, i.ModTime, precision) dt, ok := CheckTimeEqualWithPrecision(modTime, i.ModTime, precision)
if !ok { if !ok {
@ -48,6 +49,7 @@ func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, prec
} }
} }
// Check checks all the attributes of the object are correct
func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) { func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
if obj == nil { if obj == nil {
t.Fatalf("Object is nil") t.Fatalf("Object is nil")
@ -66,14 +68,14 @@ func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
i.CheckModTime(t, obj, obj.ModTime(), precision) i.CheckModTime(t, obj, obj.ModTime(), precision)
} }
// Represents all items for checking // Items represents all items for checking
type Items struct { type Items struct {
byName map[string]*Item byName map[string]*Item
byNameAlt map[string]*Item byNameAlt map[string]*Item
items []Item items []Item
} }
// Make an Items // NewItems makes an Items
func NewItems(items []Item) *Items { func NewItems(items []Item) *Items {
is := &Items{ is := &Items{
byName: make(map[string]*Item), byName: make(map[string]*Item),
@ -88,7 +90,7 @@ func NewItems(items []Item) *Items {
return is return is
} }
// Check off an item // Find checks off an item
func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) { func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
i, ok := is.byName[obj.Remote()] i, ok := is.byName[obj.Remote()]
if !ok { if !ok {
@ -103,7 +105,7 @@ func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
i.Check(t, obj, precision) i.Check(t, obj, precision)
} }
// Check all done // Done checks all finished
func (is *Items) Done(t *testing.T) { func (is *Items) Done(t *testing.T) {
if len(is.byName) != 0 { if len(is.byName) != 0 {
for name := range is.byName { for name := range is.byName {
@ -113,7 +115,8 @@ func (is *Items) Done(t *testing.T) {
} }
} }
// Checks the fs to see if it has the expected contents // CheckListingWithPrecision checks the fs to see if it has the
// expected contents with the given precision.
func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) { func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items) is := NewItems(items)
oldErrors := fs.Stats.GetErrors() oldErrors := fs.Stats.GetErrors()
@ -143,13 +146,13 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti
} }
} }
// Checks the fs to see if it has the expected contents // CheckListing checks the fs to see if it has the expected contents
func CheckListing(t *testing.T, f fs.Fs, items []Item) { func CheckListing(t *testing.T, f fs.Fs, items []Item) {
precision := f.Precision() precision := f.Precision()
CheckListingWithPrecision(t, f, items, precision) CheckListingWithPrecision(t, f, items, precision)
} }
// Parse a time string or explode // Time parses a time string or logs a fatal error
func Time(timeString string) time.Time { func Time(timeString string) time.Time {
t, err := time.Parse(time.RFC3339Nano, timeString) t, err := time.Parse(time.RFC3339Nano, timeString)
if err != nil { if err != nil {
@ -158,7 +161,7 @@ func Time(timeString string) time.Time {
return t return t
} }
// Create a random string // RandomString create a random string for test purposes
func RandomString(n int) string { func RandomString(n int) string {
source := "abcdefghijklmnopqrstuvwxyz0123456789" source := "abcdefghijklmnopqrstuvwxyz0123456789"
out := make([]byte, n) out := make([]byte, n)
@ -168,7 +171,7 @@ func RandomString(n int) string {
return string(out) return string(out)
} }
// Creates a temporary directory name for local remotes // LocalRemote creates a temporary directory name for local remotes
func LocalRemote() (path string, err error) { func LocalRemote() (path string, err error) {
path, err = ioutil.TempDir("", "rclone") path, err = ioutil.TempDir("", "rclone")
if err == nil { if err == nil {
@ -179,7 +182,7 @@ func LocalRemote() (path string, err error) {
return return
} }
// Make a random bucket or subdirectory name // RandomRemoteName makes a random bucket or subdirectory name
// //
// Returns a random remote name plus the leaf name // Returns a random remote name plus the leaf name
func RandomRemoteName(remoteName string) (string, string, error) { func RandomRemoteName(remoteName string) (string, string, error) {
@ -202,7 +205,7 @@ func RandomRemoteName(remoteName string) (string, string, error) {
return remoteName, leafName, nil return remoteName, leafName, nil
} }
// Make a random bucket or subdirectory on the remote // RandomRemote makes a random bucket or subdirectory on the remote
// //
// Call the finalise function returned to Purge the fs at the end (and // Call the finalise function returned to Purge the fs at the end (and
// the parent if necessary) // the parent if necessary)
@ -241,6 +244,7 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
return remote, finalise, nil return remote, finalise, nil
} }
// TestMkdir tests Mkdir works
func TestMkdir(t *testing.T, remote fs.Fs) { func TestMkdir(t *testing.T, remote fs.Fs) {
err := fs.Mkdir(remote) err := fs.Mkdir(remote)
if err != nil { if err != nil {
@ -249,6 +253,7 @@ func TestMkdir(t *testing.T, remote fs.Fs) {
CheckListing(t, remote, []Item{}) CheckListing(t, remote, []Item{})
} }
// TestPurge tests Purge works
func TestPurge(t *testing.T, remote fs.Fs) { func TestPurge(t *testing.T, remote fs.Fs) {
err := fs.Purge(remote) err := fs.Purge(remote)
if err != nil { if err != nil {
@ -257,6 +262,7 @@ func TestPurge(t *testing.T, remote fs.Fs) {
CheckListing(t, remote, []Item{}) CheckListing(t, remote, []Item{})
} }
// TestRmdir tests Rmdir works
func TestRmdir(t *testing.T, remote fs.Fs) { func TestRmdir(t *testing.T, remote fs.Fs) {
err := fs.Rmdir(remote) err := fs.Rmdir(remote)
if err != nil { if err != nil {

60
fstest/fstests/fstests.go
View File

@ -1,9 +1,9 @@
// Generic tests for testing the Fs and Object interfaces // Package fstests provides generic tests for testing the Fs and Object interfaces
// //
// Run go generate to write the tests for the remotes // Run go generate to write the tests for the remotes
package fstests
//go:generate go run gen_tests.go //go:generate go run gen_tests.go
package fstests
import ( import (
"bytes" "bytes"
@ -22,12 +22,14 @@ import (
) )
var ( var (
remote fs.Fs remote fs.Fs
// RemoteName should be set to the name of the remote for testing
RemoteName = "" RemoteName = ""
subRemoteName = "" subRemoteName = ""
subRemoteLeaf = "" subRemoteLeaf = ""
NilObject fs.Object // NilObject should be set to a nil Object from the Fs under test
file1 = fstest.Item{ NilObject fs.Object
file1 = fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: "file name.txt", Path: "file name.txt",
} }
@ -42,6 +44,7 @@ func init() {
flag.StringVar(&RemoteName, "remote", "", "Set this to override the default remote name (eg s3:)") flag.StringVar(&RemoteName, "remote", "", "Set this to override the default remote name (eg s3:)")
} }
// TestInit tests basic intitialisation
func TestInit(t *testing.T) { func TestInit(t *testing.T) {
var err error var err error
fs.LoadConfig() fs.LoadConfig()
@ -60,7 +63,7 @@ func TestInit(t *testing.T) {
} }
remote, err = fs.NewFs(subRemoteName) remote, err = fs.NewFs(subRemoteName)
if err == fs.NotFoundInConfigFile { if err == fs.ErrorNotFoundInConfigFile {
log.Printf("Didn't find %q in config file - skipping tests", RemoteName) log.Printf("Didn't find %q in config file - skipping tests", RemoteName)
return return
} }
@ -76,8 +79,7 @@ func skipIfNotOk(t *testing.T) {
} }
} }
// String returns a description of the FS // TestFsString tests the String method
func TestFsString(t *testing.T) { func TestFsString(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
str := remote.String() str := remote.String()
@ -86,18 +88,13 @@ func TestFsString(t *testing.T) {
} }
} }
type TestFile struct { // TestFsRmdirEmpty tests deleting an empty directory
ModTime time.Time
Path string
Size int64
Md5sum string
}
func TestFsRmdirEmpty(t *testing.T) { func TestFsRmdirEmpty(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestRmdir(t, remote) fstest.TestRmdir(t, remote)
} }
// TestFsRmdirNotFound tests deleting a non existent directory
func TestFsRmdirNotFound(t *testing.T) { func TestFsRmdirNotFound(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
err := remote.Rmdir() err := remote.Rmdir()
@ -106,17 +103,20 @@ func TestFsRmdirNotFound(t *testing.T) {
} }
} }
// TestFsMkdir tests tests making a directory
func TestFsMkdir(t *testing.T) { func TestFsMkdir(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestMkdir(t, remote) fstest.TestMkdir(t, remote)
fstest.TestMkdir(t, remote) fstest.TestMkdir(t, remote)
} }
// TestFsListEmpty tests listing an empty directory
func TestFsListEmpty(t *testing.T) { func TestFsListEmpty(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{}) fstest.CheckListing(t, remote, []fstest.Item{})
} }
// TestFsListDirEmpty tests listing the directories from an empty directory
func TestFsListDirEmpty(t *testing.T) { func TestFsListDirEmpty(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
for obj := range remote.ListDir() { for obj := range remote.ListDir() {
@ -124,6 +124,7 @@ func TestFsListDirEmpty(t *testing.T) {
} }
} }
// TestFsNewFsObjectNotFound tests not finding a object
func TestFsNewFsObjectNotFound(t *testing.T) { func TestFsNewFsObjectNotFound(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if remote.NewFsObject("potato") != nil { if remote.NewFsObject("potato") != nil {
@ -156,16 +157,19 @@ func testPut(t *testing.T, file *fstest.Item) {
file.Check(t, obj, remote.Precision()) file.Check(t, obj, remote.Precision())
} }
// TestFsPutFile1 tests putting a file
func TestFsPutFile1(t *testing.T) { func TestFsPutFile1(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
testPut(t, &file1) testPut(t, &file1)
} }
// TestFsPutFile2 tests putting a file into a subdirectory
func TestFsPutFile2(t *testing.T) { func TestFsPutFile2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
testPut(t, &file2) testPut(t, &file2)
} }
// TestFsListDirFile2 tests the files are correctly uploaded
func TestFsListDirFile2(t *testing.T) { func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
found := false found := false
@ -181,6 +185,7 @@ func TestFsListDirFile2(t *testing.T) {
} }
} }
// TestFsListDirRoot tests that DirList works in the root
func TestFsListDirRoot(t *testing.T) { func TestFsListDirRoot(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName) rootRemote, err := fs.NewFs(RemoteName)
@ -198,6 +203,7 @@ func TestFsListDirRoot(t *testing.T) {
} }
} }
// TestFsListRoot tests List works in the root
func TestFsListRoot(t *testing.T) { func TestFsListRoot(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName) rootRemote, err := fs.NewFs(RemoteName)
@ -237,22 +243,26 @@ func TestFsListRoot(t *testing.T) {
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count) t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count)
} }
// TestFsListFile1 tests file present
func TestFsListFile1(t *testing.T) { func TestFsListFile1(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
} }
// TestFsNewFsObject tests NewFsObject
func TestFsNewFsObject(t *testing.T) { func TestFsNewFsObject(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, remote.Precision())
} }
// TestFsListFile1and2 tests two files present
func TestFsListFile1and2(t *testing.T) { func TestFsListFile1and2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(t, remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
} }
// TestFsCopy tests Copy
func TestFsCopy(t *testing.T) { func TestFsCopy(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
@ -288,6 +298,7 @@ func TestFsCopy(t *testing.T) {
} }
// TestFsMove tests Move
func TestFsMove(t *testing.T) { func TestFsMove(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
@ -333,6 +344,8 @@ func TestFsMove(t *testing.T) {
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
// TestFsDirMove tests DirMove
func TestFsDirMove(t *testing.T) { func TestFsDirMove(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
@ -377,6 +390,7 @@ func TestFsDirMove(t *testing.T) {
fstest.CheckListing(t, newRemote, []fstest.Item{}) fstest.CheckListing(t, newRemote, []fstest.Item{})
} }
// TestFsRmdirFull tests removing a non empty directory
func TestFsRmdirFull(t *testing.T) { func TestFsRmdirFull(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
err := remote.Rmdir() err := remote.Rmdir()
@ -385,6 +399,7 @@ func TestFsRmdirFull(t *testing.T) {
} }
} }
// TestFsPrecision tests the Precision of the Fs
func TestFsPrecision(t *testing.T) { func TestFsPrecision(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
precision := remote.Precision() precision := remote.Precision()
@ -397,6 +412,7 @@ func TestFsPrecision(t *testing.T) {
// FIXME check expected precision // FIXME check expected precision
} }
// TestObjectString tests the Object String method
func TestObjectString(t *testing.T) { func TestObjectString(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -411,6 +427,7 @@ func TestObjectString(t *testing.T) {
} }
} }
// TestObjectFs tests the object can be found
func TestObjectFs(t *testing.T) { func TestObjectFs(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -419,6 +436,7 @@ func TestObjectFs(t *testing.T) {
} }
} }
// TestObjectRemote tests the Remote is correct
func TestObjectRemote(t *testing.T) { func TestObjectRemote(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -427,6 +445,7 @@ func TestObjectRemote(t *testing.T) {
} }
} }
// TestObjectMd5sum tests the MD5SUM of the object is correct
func TestObjectMd5sum(t *testing.T) { func TestObjectMd5sum(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -439,12 +458,14 @@ func TestObjectMd5sum(t *testing.T) {
} }
} }
// TestObjectModTime tests the ModTime of the object is correct
func TestObjectModTime(t *testing.T) { func TestObjectModTime(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision()) file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
} }
// TestObjectSetModTime tests that SetModTime works
func TestObjectSetModTime(t *testing.T) { func TestObjectSetModTime(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
@ -456,6 +477,7 @@ func TestObjectSetModTime(t *testing.T) {
TestObjectModTime(t) TestObjectModTime(t)
} }
// TestObjectSize tests that Size works
func TestObjectSize(t *testing.T) { func TestObjectSize(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -464,6 +486,7 @@ func TestObjectSize(t *testing.T) {
} }
} }
// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) { func TestObjectOpen(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -489,6 +512,7 @@ func TestObjectOpen(t *testing.T) {
} }
} }
// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) { func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
buf := bytes.NewBufferString(fstest.RandomString(200)) buf := bytes.NewBufferString(fstest.RandomString(200))
@ -508,6 +532,7 @@ func TestObjectUpdate(t *testing.T) {
file1.Check(t, obj, remote.Precision()) file1.Check(t, obj, remote.Precision())
} }
// TestObjectStorable tests that Storable works
func TestObjectStorable(t *testing.T) { func TestObjectStorable(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -516,6 +541,7 @@ func TestObjectStorable(t *testing.T) {
} }
} }
// TestLimitedFs tests that a LimitedFs is created
func TestLimitedFs(t *testing.T) { func TestLimitedFs(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
remoteName := subRemoteName + "/" + file2.Path remoteName := subRemoteName + "/" + file2.Path
@ -532,6 +558,7 @@ func TestLimitedFs(t *testing.T) {
} }
} }
// TestLimitedFsNotFound tests that a LimitedFs is not created if no object
func TestLimitedFsNotFound(t *testing.T) { func TestLimitedFsNotFound(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
remoteName := subRemoteName + "/not found.txt" remoteName := subRemoteName + "/not found.txt"
@ -546,6 +573,7 @@ func TestLimitedFsNotFound(t *testing.T) {
} }
} }
// TestObjectRemove tests Remove
func TestObjectRemove(t *testing.T) { func TestObjectRemove(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
@ -556,6 +584,7 @@ func TestObjectRemove(t *testing.T) {
fstest.CheckListing(t, remote, []fstest.Item{file2}) fstest.CheckListing(t, remote, []fstest.Item{file2})
} }
// TestObjectPurge tests Purge
func TestObjectPurge(t *testing.T) { func TestObjectPurge(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestPurge(t, remote) fstest.TestPurge(t, remote)
@ -565,6 +594,7 @@ func TestObjectPurge(t *testing.T) {
} }
} }
// TestFinalise tidies up after the previous tests
func TestFinalise(t *testing.T) { func TestFinalise(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
if strings.HasPrefix(RemoteName, "/") { if strings.HasPrefix(RemoteName, "/") {

29
googlecloudstorage/googlecloudstorage.go
View File

@ -1,4 +1,4 @@
// Google Cloud Storage interface // Package googlecloudstorage provides an interface to Google Cloud Storage
package googlecloudstorage package googlecloudstorage
/* /*
@ -55,7 +55,7 @@ var (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "google cloud storage", Name: "google cloud storage",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string) { Config: func(name string) {
@ -144,12 +144,12 @@ type FsObjectStorage struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsStorage) Name() string { func (f *FsStorage) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsStorage) Root() string { func (f *FsStorage) Root() string {
if f.root == "" { if f.root == "" {
return f.bucket return f.bucket
@ -254,7 +254,7 @@ func (f *FsStorage) newFsObjectWithInfo(remote string, info *storage.Object) fs.
return o return o
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsStorage) NewFsObject(remote string) fs.Object { func (f *FsStorage) NewFsObject(remote string) fs.Object {
@ -302,7 +302,7 @@ func (f *FsStorage) list(directories bool, fn func(string, *storage.Object)) {
} }
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsStorage) List() fs.ObjectsChan { func (f *FsStorage) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
@ -324,7 +324,7 @@ func (f *FsStorage) List() fs.ObjectsChan {
return out return out
} }
// Lists the buckets // ListDir lists the buckets
func (f *FsStorage) ListDir() fs.DirChan { func (f *FsStorage) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
@ -412,8 +412,8 @@ func (f *FsStorage) Rmdir() error {
return f.svc.Buckets.Delete(f.bucket).Do() return f.svc.Buckets.Delete(f.bucket).Do()
} }
// Return the precision // Precision returns the precision
func (fs *FsStorage) Precision() time.Duration { func (f *FsStorage) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -451,7 +451,7 @@ func (f *FsStorage) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectStorage) Fs() fs.Fs { func (o *FsObjectStorage) Fs() fs.Fs {
return o.storage return o.storage
} }
@ -464,7 +464,7 @@ func (o *FsObjectStorage) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectStorage) Remote() string { func (o *FsObjectStorage) Remote() string {
return o.remote return o.remote
} }
@ -499,9 +499,8 @@ func (o *FsObjectStorage) setMetaData(info *storage.Object) {
if err == nil { if err == nil {
o.modTime = modTime o.modTime = modTime
return return
} else {
fs.Debug(o, "Failed to read mtime from metadata: %s", err)
} }
fs.Debug(o, "Failed to read mtime from metadata: %s", err)
} }
// Fallback to the Updated time // Fallback to the Updated time
@ -549,7 +548,7 @@ func metadataFromModTime(modTime time.Time) map[string]string {
return metadata return metadata
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectStorage) SetModTime(modTime time.Time) { func (o *FsObjectStorage) SetModTime(modTime time.Time) {
// This only adds metadata so will perserve other metadata // This only adds metadata so will perserve other metadata
object := storage.Object{ object := storage.Object{
@ -565,7 +564,7 @@ func (o *FsObjectStorage) SetModTime(modTime time.Time) {
o.setMetaData(newObject) o.setMetaData(newObject)
} }
// Is this object storable // Storable returns a boolean as to whether this object is storable
func (o *FsObjectStorage) Storable() bool { func (o *FsObjectStorage) Storable() bool {
return true return true
} }

45
local/local.go
View File

@ -1,4 +1,4 @@
// Local filesystem interface // Package local provides a filesystem interface
package local package local
import ( import (
@ -10,19 +10,19 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"runtime"
"strings"
"sync" "sync"
"time" "time"
"unicode/utf8" "unicode/utf8"
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
"regexp"
"runtime"
"strings"
) )
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "local", Name: "local",
NewFs: NewFs, NewFs: NewFs,
}) })
@ -71,12 +71,12 @@ func NewFs(name, root string) (fs.Fs, error) {
return f, nil return f, nil
} }
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsLocal) Name() string { func (f *FsLocal) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsLocal) Root() string { func (f *FsLocal) Root() string {
return f.root return f.root
} }
@ -110,7 +110,7 @@ func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object
return o return o
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object { func (f *FsLocal) NewFsObject(remote string) fs.Object {
@ -195,7 +195,7 @@ func (f *FsLocal) cleanUtf8(name string) string {
return name return name
} }
// Walk the path returning a channel of FsObjects // ListDir walks the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan { func (f *FsLocal) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
go func() { go func() {
@ -220,7 +220,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err) fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err)
} else { } else {
dir.Count += 1 dir.Count++
dir.Bytes += fi.Size() dir.Bytes += fi.Size()
} }
return nil return nil
@ -238,7 +238,7 @@ func (f *FsLocal) ListDir() fs.DirChan {
return out return out
} }
// Puts the FsObject to the local filesystem // Put the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
// Temporary FsObject under construction - info filled in by Update() // Temporary FsObject under construction - info filled in by Update()
o := f.newFsObject(remote) o := f.newFsObject(remote)
@ -262,7 +262,7 @@ func (f *FsLocal) Rmdir() error {
return os.Remove(f.root) return os.Remove(f.root)
} }
// Return the precision // Precision of the file system
func (f *FsLocal) Precision() (precision time.Duration) { func (f *FsLocal) Precision() (precision time.Duration) {
f.precisionOk.Do(func() { f.precisionOk.Do(func() {
f.precision = f.readPrecision() f.precision = f.readPrecision()
@ -347,7 +347,7 @@ func (f *FsLocal) Purge() error {
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantMove // If it isn't possible then return fs.ErrorCantMove
func (dstFs *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) { func (f *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*FsObjectLocal) srcObj, ok := src.(*FsObjectLocal)
if !ok { if !ok {
fs.Debug(src, "Can't move - not same remote type") fs.Debug(src, "Can't move - not same remote type")
@ -355,7 +355,7 @@ func (dstFs *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) {
} }
// Temporary FsObject under construction // Temporary FsObject under construction
dstObj := dstFs.newFsObject(remote) dstObj := f.newFsObject(remote)
// Check it is a file if it exists // Check it is a file if it exists
err := dstObj.lstat() err := dstObj.lstat()
@ -389,14 +389,15 @@ func (dstFs *FsLocal) Move(src fs.Object, remote string) (fs.Object, error) {
return dstObj, nil return dstObj, nil
} }
// Move src to this remote using server side move operations. // DirMove moves src directory to this remote using server side move
// operations.
// //
// Will only be called if src.Fs().Name() == f.Name() // Will only be called if src.Fs().Name() == f.Name()
// //
// If it isn't possible then return fs.ErrorCantDirMove // If it isn't possible then return fs.ErrorCantDirMove
// //
// If destination exists then return fs.ErrorDirExists // If destination exists then return fs.ErrorDirExists
func (dstFs *FsLocal) DirMove(src fs.Fs) error { func (f *FsLocal) DirMove(src fs.Fs) error {
srcFs, ok := src.(*FsLocal) srcFs, ok := src.(*FsLocal)
if !ok { if !ok {
fs.Debug(srcFs, "Can't move directory - not same remote type") fs.Debug(srcFs, "Can't move directory - not same remote type")
@ -413,18 +414,18 @@ func (dstFs *FsLocal) DirMove(src fs.Fs) error {
} }
// Check if destination exists // Check if destination exists
_, err = os.Lstat(dstFs.root) _, err = os.Lstat(f.root)
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
return fs.ErrorDirExists return fs.ErrorDirExists
} }
// Do the move // Do the move
return os.Rename(srcFs.root, dstFs.root) return os.Rename(srcFs.root, f.root)
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs { func (o *FsObjectLocal) Fs() fs.Fs {
return o.local return o.local
} }
@ -437,7 +438,7 @@ func (o *FsObjectLocal) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectLocal) Remote() string { func (o *FsObjectLocal) Remote() string {
return o.local.cleanUtf8(o.remote) return o.local.cleanUtf8(o.remote)
} }
@ -480,7 +481,7 @@ func (o *FsObjectLocal) ModTime() time.Time {
return o.info.ModTime() return o.info.ModTime()
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) { func (o *FsObjectLocal) SetModTime(modTime time.Time) {
err := os.Chtimes(o.path, modTime, modTime) err := os.Chtimes(o.path, modTime, modTime)
if err != nil { if err != nil {
@ -495,7 +496,7 @@ func (o *FsObjectLocal) SetModTime(modTime time.Time) {
} }
} }
// Is this object storable // Storable returns a boolean showing if this object is storable
func (o *FsObjectLocal) Storable() bool { func (o *FsObjectLocal) Storable() bool {
mode := o.info.Mode() mode := o.info.Mode()
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {

14
oauthutil/oauthutil.go
View File

@ -210,24 +210,24 @@ func Config(name string, config *oauth2.Config) error {
return err return err
} }
state := fmt.Sprintf("%x", stateBytes) state := fmt.Sprintf("%x", stateBytes)
authUrl := config.AuthCodeURL(state) authURL := config.AuthCodeURL(state)
// Prepare webserver // Prepare webserver
server := authServer{ server := authServer{
state: state, state: state,
bindAddress: bindAddress, bindAddress: bindAddress,
authUrl: authUrl, authURL: authURL,
} }
if useWebServer { if useWebServer {
server.code = make(chan string, 1) server.code = make(chan string, 1)
go server.Start() go server.Start()
defer server.Stop() defer server.Stop()
authUrl = "http://" + bindAddress + "/auth" authURL = "http://" + bindAddress + "/auth"
} }
// Generate a URL for the user to visit for authorization. // Generate a URL for the user to visit for authorization.
_ = open.Start(authUrl) _ = open.Start(authURL)
fmt.Printf("If your browser doesn't open automatically go to the following link: %s\n", authUrl) fmt.Printf("If your browser doesn't open automatically go to the following link: %s\n", authURL)
fmt.Printf("Log in and authorize rclone for access\n") fmt.Printf("Log in and authorize rclone for access\n")
var authCode string var authCode string
@ -258,7 +258,7 @@ type authServer struct {
listener net.Listener listener net.Listener
bindAddress string bindAddress string
code chan string code chan string
authUrl string authURL string
} }
// startWebServer runs an internal web server to receive config details // startWebServer runs an internal web server to receive config details
@ -274,7 +274,7 @@ func (s *authServer) Start() {
return return
}) })
mux.HandleFunc("/auth", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/auth", func(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, s.authUrl, 307) http.Redirect(w, req, s.authURL, 307)
return return
}) })
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {

10
rclone.go
View File

@ -35,6 +35,7 @@ var (
retries = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail") retries = pflag.IntP("retries", "", 3, "Retry operations this many times if they fail")
) )
// Command holds info about the current running command
type Command struct { type Command struct {
Name string Name string
Help string Help string
@ -59,6 +60,7 @@ func (cmd *Command) checkArgs(args []string) {
} }
} }
// Commands is a slice of possible Command~s
var Commands = []Command{ var Commands = []Command{
{ {
Name: "copy", Name: "copy",
@ -249,7 +251,7 @@ func fatal(message string, args ...interface{}) {
os.Exit(1) os.Exit(1)
} }
// Parse the command line flags // ParseFlags parses the command line flags
func ParseFlags() { func ParseFlags() {
pflag.Usage = syntaxError pflag.Usage = syntaxError
pflag.Parse() pflag.Parse()
@ -272,7 +274,7 @@ func ParseFlags() {
} }
} }
// Parse the command from the command line // ParseCommand parses the command from the command line
func ParseCommand() (*Command, []string) { func ParseCommand() (*Command, []string) {
args := pflag.Args() args := pflag.Args()
if len(args) < 1 { if len(args) < 1 {
@ -318,7 +320,7 @@ func ParseCommand() (*Command, []string) {
return command, args return command, args
} }
// Create a Fs from a name // NewFs creates a Fs from a name
func NewFs(remote string) fs.Fs { func NewFs(remote string) fs.Fs {
f, err := fs.NewFs(remote) f, err := fs.NewFs(remote)
if err != nil { if err != nil {
@ -328,7 +330,7 @@ func NewFs(remote string) fs.Fs {
return f return f
} }
// Print the stats every statsInterval // StartStats prints the stats every statsInterval
func StartStats() { func StartStats() {
if *statsInterval <= 0 { if *statsInterval <= 0 {
return return

34
s3/s3.go
View File

@ -1,4 +1,4 @@
// S3 interface // Package s3 provides an interface to Amazon S3 oject storage
package s3 package s3
// FIXME need to prevent anything but ListDir working for s3:// // FIXME need to prevent anything but ListDir working for s3://
@ -35,7 +35,7 @@ import (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "s3", Name: "s3",
NewFs: NewFs, NewFs: NewFs,
// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region // AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
@ -153,12 +153,12 @@ type FsObjectS3 struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsS3) Name() string { func (f *FsS3) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsS3) Root() string { func (f *FsS3) Root() string {
if f.root == "" { if f.root == "" {
return f.bucket return f.bucket
@ -192,15 +192,15 @@ func s3ParsePath(path string) (bucket, directory string, err error) {
// s3Connection makes a connection to s3 // s3Connection makes a connection to s3
func s3Connection(name string) (*s3.S3, error) { func s3Connection(name string) (*s3.S3, error) {
// Make the auth // Make the auth
accessKeyId := fs.ConfigFile.MustValue(name, "access_key_id") accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
if accessKeyId == "" { if accessKeyID == "" {
return nil, errors.New("access_key_id not found") return nil, errors.New("access_key_id not found")
} }
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key") secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
if secretAccessKey == "" { if secretAccessKey == "" {
return nil, errors.New("secret_access_key not found") return nil, errors.New("secret_access_key not found")
} }
auth := credentials.NewStaticCredentials(accessKeyId, secretAccessKey, "") auth := credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
endpoint := fs.ConfigFile.MustValue(name, "endpoint") endpoint := fs.ConfigFile.MustValue(name, "endpoint")
region := fs.ConfigFile.MustValue(name, "region") region := fs.ConfigFile.MustValue(name, "region")
@ -226,7 +226,7 @@ func s3Connection(name string) (*s3.S3, error) {
if req.Service.Config.Credentials == credentials.AnonymousCredentials { if req.Service.Config.Credentials == credentials.AnonymousCredentials {
return return
} }
sign(accessKeyId, secretAccessKey, req.HTTPRequest) sign(accessKeyID, secretAccessKey, req.HTTPRequest)
} }
c.Handlers.Sign.Clear() c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
@ -239,7 +239,7 @@ func s3Connection(name string) (*s3.S3, error) {
return c, nil return c, nil
} }
// NewFsS3 contstructs an FsS3 from the path, bucket:path // NewFs contstructs an FsS3 from the path, bucket:path
func NewFs(name, root string) (fs.Fs, error) { func NewFs(name, root string) (fs.Fs, error) {
bucket, directory, err := s3ParsePath(root) bucket, directory, err := s3ParsePath(root)
if err != nil { if err != nil {
@ -310,7 +310,7 @@ func (f *FsS3) newFsObjectWithInfo(remote string, info *s3.Object) fs.Object {
return o return o
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsS3) NewFsObject(remote string) fs.Object { func (f *FsS3) NewFsObject(remote string) fs.Object {
@ -384,7 +384,7 @@ func (f *FsS3) list(directories bool, fn func(string, *s3.Object)) {
} }
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsS3) List() fs.ObjectsChan { func (f *FsS3) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
@ -405,7 +405,7 @@ func (f *FsS3) List() fs.ObjectsChan {
return out return out
} }
// Lists the buckets // ListDir lists the buckets
func (f *FsS3) ListDir() fs.DirChan { func (f *FsS3) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.bucket == "" { if f.bucket == "" {
@ -486,7 +486,7 @@ func (f *FsS3) Rmdir() error {
return err return err
} }
// Return the precision // Precision of the remote
func (f *FsS3) Precision() time.Duration { func (f *FsS3) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -524,7 +524,7 @@ func (f *FsS3) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectS3) Fs() fs.Fs { func (o *FsObjectS3) Fs() fs.Fs {
return o.s3 return o.s3
} }
@ -537,7 +537,7 @@ func (o *FsObjectS3) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectS3) Remote() string { func (o *FsObjectS3) Remote() string {
return o.remote return o.remote
} }
@ -619,7 +619,7 @@ func (o *FsObjectS3) ModTime() time.Time {
return modTime return modTime
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectS3) SetModTime(modTime time.Time) { func (o *FsObjectS3) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
@ -648,7 +648,7 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
} }
} }
// Is this object storable // Storable raturns a boolean indicating if this object is storable
func (o *FsObjectS3) Storable() bool { func (o *FsObjectS3) Storable() bool {
return true return true
} }

32
swift/swift.go
View File

@ -1,4 +1,4 @@
// Swift interface // Package swift provides an interface to the Swift object storage system
package swift package swift
import ( import (
@ -16,7 +16,7 @@ import (
// Register with Fs // Register with Fs
func init() { func init() {
fs.Register(&fs.FsInfo{ fs.Register(&fs.Info{
Name: "swift", Name: "swift",
NewFs: NewFs, NewFs: NewFs,
Options: []fs.Option{{ Options: []fs.Option{{
@ -76,12 +76,12 @@ type FsObjectSwift struct {
// ------------------------------------------------------------ // ------------------------------------------------------------
// The name of the remote (as passed into NewFs) // Name of the remote (as passed into NewFs)
func (f *FsSwift) Name() string { func (f *FsSwift) Name() string {
return f.name return f.name
} }
// The root of the remote (as passed into NewFs) // Root of the remote (as passed into NewFs)
func (f *FsSwift) Root() string { func (f *FsSwift) Root() string {
if f.root == "" { if f.root == "" {
return f.container return f.container
@ -122,14 +122,14 @@ func swiftConnection(name string) (*swift.Connection, error) {
if apiKey == "" { if apiKey == "" {
return nil, errors.New("key not found") return nil, errors.New("key not found")
} }
authUrl := fs.ConfigFile.MustValue(name, "auth") authURL := fs.ConfigFile.MustValue(name, "auth")
if authUrl == "" { if authURL == "" {
return nil, errors.New("auth not found") return nil, errors.New("auth not found")
} }
c := &swift.Connection{ c := &swift.Connection{
UserName: userName, UserName: userName,
ApiKey: apiKey, ApiKey: apiKey,
AuthUrl: authUrl, AuthUrl: authURL,
UserAgent: fs.UserAgent, UserAgent: fs.UserAgent,
Tenant: fs.ConfigFile.MustValue(name, "tenant"), Tenant: fs.ConfigFile.MustValue(name, "tenant"),
Region: fs.ConfigFile.MustValue(name, "region"), Region: fs.ConfigFile.MustValue(name, "region"),
@ -201,7 +201,7 @@ func (f *FsSwift) newFsObjectWithInfo(remote string, info *swift.Object) fs.Obje
return fs return fs
} }
// Return an FsObject from a path // NewFsObject returns an FsObject from a path
// //
// May return nil if an error occurred // May return nil if an error occurred
func (f *FsSwift) NewFsObject(remote string) fs.Object { func (f *FsSwift) NewFsObject(remote string) fs.Object {
@ -249,7 +249,7 @@ func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
} }
} }
// Walk the path returning a channel of FsObjects // List walks the path returning a channel of FsObjects
func (f *FsSwift) List() fs.ObjectsChan { func (f *FsSwift) List() fs.ObjectsChan {
out := make(fs.ObjectsChan, fs.Config.Checkers) out := make(fs.ObjectsChan, fs.Config.Checkers)
if f.container == "" { if f.container == "" {
@ -271,7 +271,7 @@ func (f *FsSwift) List() fs.ObjectsChan {
return out return out
} }
// Lists the containers // ListDir lists the containers
func (f *FsSwift) ListDir() fs.DirChan { func (f *FsSwift) ListDir() fs.DirChan {
out := make(fs.DirChan, fs.Config.Checkers) out := make(fs.DirChan, fs.Config.Checkers)
if f.container == "" { if f.container == "" {
@ -331,8 +331,8 @@ func (f *FsSwift) Rmdir() error {
return f.c.ContainerDelete(f.container) return f.c.ContainerDelete(f.container)
} }
// Return the precision // Precision of the remote
func (fs *FsSwift) Precision() time.Duration { func (f *FsSwift) Precision() time.Duration {
return time.Nanosecond return time.Nanosecond
} }
@ -361,7 +361,7 @@ func (f *FsSwift) Copy(src fs.Object, remote string) (fs.Object, error) {
// ------------------------------------------------------------ // ------------------------------------------------------------
// Return the parent Fs // Fs returns the parent Fs
func (o *FsObjectSwift) Fs() fs.Fs { func (o *FsObjectSwift) Fs() fs.Fs {
return o.swift return o.swift
} }
@ -374,7 +374,7 @@ func (o *FsObjectSwift) String() string {
return o.remote return o.remote
} }
// Return the remote path // Remote returns the remote path
func (o *FsObjectSwift) Remote() string { func (o *FsObjectSwift) Remote() string {
return o.remote return o.remote
} }
@ -426,7 +426,7 @@ func (o *FsObjectSwift) ModTime() time.Time {
return modTime return modTime
} }
// Sets the modification time of the local fs object // SetModTime sets the modification time of the local fs object
func (o *FsObjectSwift) SetModTime(modTime time.Time) { func (o *FsObjectSwift) SetModTime(modTime time.Time) {
err := o.readMetaData() err := o.readMetaData()
if err != nil { if err != nil {
@ -442,7 +442,7 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
} }
} }
// Is this object storable // Storable returns if this object is storable
func (o *FsObjectSwift) Storable() bool { func (o *FsObjectSwift) Storable() bool {
return true return true
} }