yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add ftp backend - fixes #540

s3-about
Antonio Messina 2016-12-27 21:52:30 +01:00 committed by Nick Craig-Wood
parent ae17d88518
commit c72fca2711
11 changed files with 574 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -25,6 +25,7 @@ Rclone is a command line program to sync files and directories to and from
* Yandex Disk
* SFTP
* The local filesystem
* FTP
Features

1
docs/content/about.md
View File

@ -25,6 +25,7 @@ Rclone is a command line program to sync files and directories to and from
* Yandex Disk
* SFTP
* The local filesystem
* FTP
Features

1
docs/content/docs.md
View File

@ -31,6 +31,7 @@ See the following for detailed instructions for
* [Microsoft OneDrive](/onedrive/)
* [Yandex Disk](/yandex/)
* [SFTP](/sftp/)
* [FTP](/ftp/)
* [Crypt](/crypt/) - to encrypt other remotes
Usage

35
docs/content/ftp.md Normal file
View File

@ -0,0 +1,35 @@
---
title: "FTP"
description: "Rclone docs for FTP backend"
date: "2017-01-01"
---
<i class="fa fa-file"></i> FTP
------------------------------
FTP support is provided via
[github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
package.
### Configuration ###
An Ftp backend only needs an Url and and username and password. With
anonymous FTP server you will need to use `anonymous` as username and
your email address as password.
Example:
```
[remote]
type = Ftp
username = anonymous
password = john.snow@example.org
url = ftp://ftp.kernel.org/pub
```
### Unsupported features ###
FTP backends does not support:
* Any hash mechanism
* Modified Time
* remote copy/move

2
docs/content/overview.md
View File

@ -29,6 +29,7 @@ Here is an overview of the major features of each cloud storage system.
| Yandex Disk | MD5 | Yes | No | No | R/W |
| SFTP | - | Yes | Depends | No | - |
| The local filesystem | All | Yes | Depends | No | - |
| FTP | None | No | Yes | No | - |
### Hash ###
@ -117,6 +118,7 @@ operations more efficient.
| Yandex Disk | Yes | No | No | No | No [#575](https://github.com/ncw/rclone/issues/575) |
| SFTP | No | No | Yes | Yes | No |
| The local filesystem | Yes | No | Yes | Yes | No |
| FTP | No | No | No | No | No |
### Purge ###

1
fs/all/all.go
View File

@ -15,4 +15,5 @@ import (
_ "github.com/ncw/rclone/sftp"
_ "github.com/ncw/rclone/swift"
_ "github.com/ncw/rclone/yandex"
_ "github.com/ncw/rclone/ftp"
)

1
fs/test_all.go
View File

@ -35,6 +35,7 @@ var (
"TestSftp:",
"TestSwift:",
"TestYandex:",
"TestFTP:",
}
binary = "fs.test"
// Flags

1
fstest/fstests/gen_tests.go
View File

@ -142,5 +142,6 @@ func main() {
generateTestProgram(t, fns, "Crypt", "2")
generateTestProgram(t, fns, "Crypt", "3")
generateTestProgram(t, fns, "Sftp", "")
generateTestProgram(t, fns, "FTP", "")
log.Printf("Done")
}

433
ftp/ftp.go Normal file
View File

@ -0,0 +1,433 @@
// Package fs is a generic file system interface for rclone object storage systems
package ftp
import (
"fmt"
"github.com/jlaffaye/ftp"
"github.com/ncw/rclone/fs"
"io"
"path/filepath"
"regexp"
"strconv"
"time"
"sync"
"strings"
)
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
Name: "Ftp",
Description: "FTP interface",
NewFs: NewFs,
Options: []fs.Option{
{
Name: "username",
Help: "Username",
}, {
Name: "password",
Help: "Password",
}, {
Name: "url",
Help: "FTP url",
},
},
})
}
type Url struct {
Scheme string
Host string
Port int
Path string
}
type Fs struct {
name string // name of this remote
c *ftp.ServerConn // the connection to the FTP server
root string // the path we are working on if any
url Url
mu sync.Mutex
}
type Object struct {
fs *Fs
remote string
info *FileInfo
}
type FileInfo struct {
Name string
Size uint64
ModTime time.Time
IsDir bool
}
// Implements ReadCloser for FTP objects.
type FtpReadCloser struct {
remote string
c *ftp.ServerConn
fd io.ReadCloser
}
/////////////////
// Url methods //
/////////////////
func (u *Url) ToDial() string {
return fmt.Sprintf("%s:%d", u.Host, u.Port)
}
func (u *Url) String() string {
return fmt.Sprintf("ftp://%s:%d/%s", u.Host, u.Port, u.Path)
}
func parseUrl(url string) Url {
// This is *similar* to the RFC 3986 regexp but it matches the
// port independently from the host
r, _ := regexp.Compile("^(([^:/?#]+):)?(//([^/?#:]*))?(:([0-9]+))?([^?#]*)(\\?([^#]*))?(#(.*))?")
data := r.FindAllStringSubmatch(url, -1)
if data[0][5] == "" { data[0][5] = "21" }
port, _ := strconv.Atoi(data[0][5])
return Url{data[0][2], data[0][4], port, data[0][7]}
}
////////////////
// Fs Methods //
////////////////
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
fs.Debug(f, "Trying to put file %s", src.Remote())
o := &Object{
fs: f,
remote: src.Remote(),
}
err := o.Update(in, src)
return o, err
}
func (f *Fs) Rmdir(dir string) error {
// This is actually a recursive remove directory
f.mu.Lock()
files, _ := f.c.List(filepath.Join(f.root, dir))
f.mu.Unlock()
for i:= range files {
if files[i].Type == ftp.EntryTypeFolder {
f.Rmdir(filepath.Join(dir, files[i].Name))
}
}
f.mu.Lock()
err := f.c.RemoveDir(filepath.Join(f.root, dir))
f.mu.Unlock()
return err
}
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
func (f *Fs) String() string {
return fmt.Sprintf("FTP Connection to %s", f.url.String())
}
// Hash are not supported
func (f *Fs) Hashes() fs.HashSet {
return 0
}
// Modified Time not supported
func (f *Fs) Precision() time.Duration {
return fs.ModTimeNotSupported
}
func (f *Fs) mkdir(abspath string) error {
_, err := f.GetInfo(abspath)
if err != nil {
fs.Debug(f, "Trying to create directory %s", abspath)
f.mu.Lock()
err := f.c.MakeDir(abspath)
f.mu.Unlock()
if err != nil {
return err
}
}
return err
}
func (f *Fs) Mkdir(dir string) error {
// This actually works as mkdir -p
fs.Debug(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir)
defer fs.Debug(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir)
abspath := filepath.Join(f.root, dir)
tokens := strings.Split(abspath, "/")
curdir := ""
for i:= range tokens {
curdir += "/" + tokens[i]
f.mkdir(curdir)
}
return nil
}
func (f *Fs) GetInfo(remote string) (*FileInfo, error) {
fs.Debug(f, "ENTER function 'GetInfo' on file %s", remote)
defer fs.Debug(f, "EXIT function 'GetInfo'")
dir := filepath.Dir(remote)
base := filepath.Base(remote)
f.mu.Lock()
files, _ := f.c.List(dir)
f.mu.Unlock()
for i:= range files {
if files[i].Name == base {
info := &FileInfo{
Name: remote,
Size: files[i].Size,
ModTime: files[i].Time,
IsDir: files[i].Type == ftp.EntryTypeFolder,
}
return info, nil
}
}
return nil, fs.ErrorObjectNotFound
}
func (f *Fs) NewObject(remote string) (fs.Object, error) {
fs.Debug(f, "ENTER function 'NewObject' called with remote %s", remote)
defer fs.Debug(f, "EXIT function 'NewObject'")
dir := filepath.Dir(remote)
base := filepath.Base(remote)
f.mu.Lock()
files, _ := f.c.List(dir)
f.mu.Unlock()
for i:= range files {
if files[i].Name == base {
o := &Object{
fs: f,
remote: remote,
}
info := &FileInfo{
Name: remote,
Size: files[i].Size,
ModTime: files[i].Time,
}
o.info = info
return o, nil
}
}
return nil, fs.ErrorObjectNotFound
}
func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
fs.Debug(f, "ENTER function 'list'")
defer fs.Debug(f, "EXIT function 'list'")
f.mu.Lock()
files, _ := f.c.List(filepath.Join(f.root, dir))
f.mu.Unlock()
for i:= range files {
object := files[i]
newremote := filepath.Join(dir, object.Name)
switch object.Type {
case ftp.EntryTypeFolder:
if out.IncludeDirectory(newremote){
d := &fs.Dir{
Name: newremote,
When: object.Time,
Bytes: 0,
Count: -1,
}
if curlevel < out.Level(){
f.list(out, filepath.Join(dir, object.Name), curlevel +1 )
}
if out.AddDir(d) {
return
}
}
default:
o := &Object{
fs: f,
remote: newremote,
}
info := &FileInfo{
Name: newremote,
Size: object.Size,
ModTime: object.Time,
}
o.info = info
if out.Add(o) {
return
}
}
}
}
func (f *Fs) List(out fs.ListOpts, dir string) {
fs.Debug(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir)
defer fs.Debug(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir)
f.list(out, dir, 1)
out.Finished()
}
////////////////////
// Object methods //
////////////////////
func (o *Object) Hash(t fs.HashType) (string, error) {
return "", fs.ErrHashUnsupported
}
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
path := filepath.Join(o.fs.root, o.remote)
fs.Debug(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root)
defer fs.Debug(o.fs, "EXIT function 'Open' %s", path)
c, _, err := ftpConnection(o.fs.name, o.fs.root)
if err != nil {
return nil, err
}
fd, err := c.Retr(path)
if err != nil {
return nil, err
}
return FtpReadCloser{path, c, fd}, nil
}
func (o *Object) Remote() string {
return o.remote
}
func (o *Object) Remove() error {
path := filepath.Join(o.fs.root, o.remote)
fs.Debug(o, "ENTER function 'Remove' for obejct at %s", path)
defer fs.Debug(o, "EXIT function 'Remove' for obejct at %s", path)
// Check if it's a directory or a file
info, _ := o.fs.GetInfo(path)
var err error
if info.IsDir {
err = o.fs.Rmdir(o.remote)
} else {
o.fs.mu.Lock()
err = o.fs.c.Delete(path)
o.fs.mu.Unlock()
}
return err
}
func (o *Object) SetModTime(modTime time.Time) error {
return nil
}
func (o *Object) Fs() fs.Info {
return o.fs
}
func (o *Object) ModTime() time.Time {
return o.info.ModTime
}
func (o *Object) Size() int64 {
return int64(o.info.Size)
}
func (o *Object) Storable() bool {
return true
}
func (o *Object) String() string {
return fmt.Sprintf("FTP file at %s/%s", o.fs.url.String(), o.remote)
}
func (o *Object) MakeAllDir() {
tokens := strings.Split(filepath.Dir(o.remote), "/")
dir := ""
for i:= range tokens {
dir += tokens[i]+"/"
o.fs.Mkdir(dir)
}
}
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
// Create all upper directory first...
o.MakeAllDir()
path := filepath.Join(o.fs.root, o.remote)
c, _, _ := ftpConnection(o.fs.name, o.fs.root)
err := c.Stor(path, in)
o.info, _ = o.fs.GetInfo(path)
return err
}
///////////////////////////
// FtpReadCloser methods //
///////////////////////////
func (f FtpReadCloser) Read(p []byte) (int, error) {
return f.fd.Read(p)
}
func (f FtpReadCloser) Close() error {
err := f.fd.Close()
defer f.c.Quit()
if err != nil {
return nil
}
return nil
}
// This mutex is only used by ftpConnection. We create a new ftp
// connection for each transfer, but we need to serialize it otherwise
// Dial() and Login() might be mixed...
var globalMux = sync.Mutex{}
func ftpConnection(name, root string) (*ftp.ServerConn, Url, error) {
// Open a new connection to the FTP server.
url := fs.ConfigFileGet(name, "url")
user := fs.ConfigFileGet(name, "username")
pass := fs.ConfigFileGet(name, "password")
u := parseUrl(url)
u.Path = filepath.Join(u.Path, root)
fs.Debug(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path)
globalMux.Lock()
defer globalMux.Unlock()
c, err := ftp.DialTimeout(u.ToDial(), 30*time.Second)
if err != nil {
fs.ErrorLog(nil, "Error while Dialing %s: %s", u.ToDial(), err)
return nil, u, err
}
err = c.Login(user, pass)
if err != nil {
fs.ErrorLog(nil, "Error while Logging in into %s: %s", u.ToDial(), err)
return nil, u, err
}
return c, u, nil
}
// Register the FS
func NewFs(name, root string) (fs.Fs, error) {
fs.Debug(nil, "ENTER function 'NewFs' with name %s and root %s", name, root)
defer fs.Debug(nil, "EXIT function 'NewFs'")
c, u, err := ftpConnection(name, root)
if err != nil {
return nil, err
}
fs := &Fs{
name: name,
root: u.Path,
c: c,
url: u,
mu: sync.Mutex{},
}
return fs, err
}
var (
_ fs.Fs = &Fs{}
)

36
ftp/ftp_internal_test.go Normal file
View File

@ -0,0 +1,36 @@
package ftp
import "testing"
func TestParseUrlToDial(t *testing.T){
for _, test := range []struct {
in string
want string
}{
{"ftp://foo.bar", "foo.bar:21"},
{"http://foo.bar", "foo.bar:21"},
{"ftp:/foo.bar:123", "foo.bar:123"},
} {
u := parseUrl(test.in)
got := u.ToDial()
if got != test.want {
t.Logf("%q: want %q got %q", test.in, test.want, got)
}
}
}
func TestParseUrlPath(t *testing.T){
for _, test := range []struct {
in string
want string
}{
{"ftp://foo.bar/", "/"},
{"ftp://foo.bar/debian", "/debian"},
{"ftp://foo.bar", "/"},
} {
u := parseUrl(test.in)
if u.Path != test.want {
t.Logf("%q: want %q got %q", test.in, test.want, u.Path)
}
}
}

62
ftp/ftp_test.go Normal file
View File

@ -0,0 +1,62 @@
// Test FTP filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: make gen_tests
package ftp_test
import (
"testing"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
"github.com/ncw/rclone/ftp"
)
func TestSetup(t *testing.T) {
fstests.NilObject = fs.Object((*ftp.Object)(nil))
fstests.RemoteName = "TestFTP:"
}
// Generic tests for the Fs
func TestInit(t *testing.T) { fstests.TestInit(t) }
func TestFsString(t *testing.T) { fstests.TestFsString(t) }
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }