yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implement Hubic storage system - fixes #200

s3-about
Nick Craig-Wood 2015-11-08 15:29:58 +00:00
parent 5023050d95
commit fcea3777c0
14 changed files with 469 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/content/about.md
View File

@ -20,6 +20,7 @@ Rclone is a command line program to sync files and directories to and from
* Google Cloud Storage
* Amazon Cloud Drive
* Microsoft One Drive
* Hubic
* The local filesystem
Features

98
docs/content/hubic.md Normal file
View File

@ -0,0 +1,98 @@
---
title: "Hubic"
description: "Rclone docs for Hubic"
date: "2015-11-08"
---
<i class="fa fa-space-shuttle"></i> Hubic
-----------------------------------------
Paths are specified as `remote:path`
Paths are specified as `remote:container` (or `remote:` for the `lsd`
command.) You may put subdirectories in too, eg `remote:container/path/to/dir`.
The initial setup for Hubic involves getting a token from Hubic which
you need to do in your browser. `rclone config` walks you through it.
Here is an example of how to make a remote called `remote`. First run:
rclone config
This will guide you through an interactive setup process:
```
n) New remote
d) Delete remote
q) Quit config
e/n/d/q> n
name> remote
What type of source is it?
Choose a number from below
1) amazon cloud drive
2) drive
3) dropbox
4) google cloud storage
5) local
6) onedrive
7) hubic
8) s3
9) swift
type> 7
Hubic App Client Id - leave blank normally.
client_id>
Hubic App Client Secret - leave blank normally.
client_secret>
Remote config
If your browser doesn't open automatically go to the following link: http://localhost:53682/auth
Log in and authorize rclone for access
Waiting for code...
Got code
--------------------
[remote]
client_id =
client_secret =
token = {"access_token":"XXXXXX"}
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
Note that rclone runs a webserver on your local machine to collect the
token as returned from Hubic. This only runs from the moment it opens
your browser to the moment you get back the verification code. This
is on `http://127.0.0.1:53682/` and this it may require you to unblock
it temporarily if you are running a host firewall.
Once configured you can then use `rclone` like this,
List containers in the top level of your Hubic
rclone lsd remote:
List all the files in your Hubic
rclone ls remote:
To copy a local directory to an Hubic directory called backup
rclone copy /home/source remote:backup
### Modified time ###
The modified time is stored as metadata on the object as
`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
ns.
This is a defacto standard (used in the official python-swiftclient
amongst others) for storing the modification time for an object.
Note that Hubic wraps the Swift backend, so most of the properties of
are the same.
### Limitations ###
Code to refresh the OpenStack token isn't done yet which may cause
problems with very long transfers.

1
docs/content/overview.md
View File

@ -24,6 +24,7 @@ Here is an overview of the major features of each cloud storage system.
| Google Cloud Storage | Yes | Yes | No | No |
| Amazon Cloud Drive | Yes | No | Yes | No |
| Microsoft One Drive | No | Yes | Yes | No |
| Hubic | Yes | Yes | No | No |
| The local filesystem | Yes | Yes | Depends | No |
### MD5SUM ###

1
docs/layouts/chrome/navbar.html
View File

@ -38,6 +38,7 @@
<li><a href="/googlecloudstorage/"><i class="fa fa-google"></i> Google Cloud Storage</a></li>
<li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li>
<li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li>
<li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
<li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
</ul>
</li>

1
fs/operations_test.go
View File

@ -24,6 +24,7 @@ import (
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/hubic"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/onedrive"
_ "github.com/ncw/rclone/s3"

1
fs/test_all.sh
View File

@ -10,6 +10,7 @@ TestGoogleCloudStorage:
TestDropbox:
TestAmazonCloudDrive:
TestOneDrive:
TestHubic:
"
function test_remote {

17
fstest/fstests/fstests.go
View File

@ -435,7 +435,14 @@ func TestObjectString(t *testing.T) {
func TestObjectFs(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
if obj.Fs() != remote {
equal := obj.Fs() == remote
if !equal {
// Check to see if this wraps something else
if unwrap, ok := remote.(fs.UnWrapper); ok {
equal = obj.Fs() == unwrap.UnWrap()
}
}
if !equal {
t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
}
}
@ -558,7 +565,13 @@ func TestLimitedFs(t *testing.T) {
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
_, ok := fileRemote.(*fs.Limited)
if !ok {
t.Errorf("%v is not a fs.Limited", fileRemote)
// Check to see if this wraps a Limited FS
if unwrap, hasUnWrap := fileRemote.(fs.UnWrapper); hasUnWrap {
_, ok = unwrap.UnWrap().(*fs.Limited)
}
if !ok {
t.Errorf("%v is not a fs.Limited", fileRemote)
}
}
}

1
fstest/fstests/gen_tests.go
View File

@ -132,5 +132,6 @@ func main() {
generateTestProgram(t, fns, "Dropbox")
generateTestProgram(t, fns, "AmazonCloudDrive")
generateTestProgram(t, fns, "OneDrive")
generateTestProgram(t, fns, "Hubic")
log.Printf("Done")
}

54
hubic/auth.go Normal file
View File

@ -0,0 +1,54 @@
package hubic
import (
"net/http"
"github.com/ncw/swift"
)
// auth is an authenticator for swift
type auth struct {
f *Fs
}
// newAuth creates a swift authenticator
func newAuth(f *Fs) *auth {
return &auth{
f: f,
}
}
// Request constructs a http.Request for authentication
//
// returns nil for not needed
func (a *auth) Request(*swift.Connection) (*http.Request, error) {
err := a.f.getCredentials()
if err != nil {
return nil, err
}
return nil, nil
}
// Response parses the result of an http request
func (a *auth) Response(resp *http.Response) error {
return nil
}
// The public storage URL - set Internal to true to read
// internal/service net URL
func (a *auth) StorageUrl(Internal bool) string {
return a.f.credentials.Endpoint
}
// The access token
func (a *auth) Token() string {
return a.f.credentials.Token
}
// The CDN url if available
func (a *auth) CdnUrl() string {
return ""
}
// Check the interfaces are satisfied
var _ swift.Authenticator = (*auth)(nil)

226
hubic/hubic.go Normal file
View File

@ -0,0 +1,226 @@
// Package hubic provides an interface to the Hubic object storage
// system.
package hubic
// This uses the normal swift mechanism to update the credentials and
// ignores the expires field returned by the Hubic API. This may need
// to be revisted after some actual experience.
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/oauthutil"
"github.com/ncw/rclone/swift"
swiftLib "github.com/ncw/swift"
"golang.org/x/oauth2"
)
const (
rcloneClientID = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
rcloneClientSecret = "8MrG3pjWyJya4OnO9ZTS4emI+9fa1ouPgvfD2MbTzfDYvO/H5czFxsTXtcji4/Hz3snz8/CrzMzlxvP9//Ty/Q=="
)
// Globals
var (
// Description of how to auth for this app
oauthConfig = &oauth2.Config{
Scopes: []string{
"credentials.r", // Read Openstack credentials
},
Endpoint: oauth2.Endpoint{
AuthURL: "https://api.hubic.com/oauth/auth/",
TokenURL: "https://api.hubic.com/oauth/token/",
},
ClientID: rcloneClientID,
ClientSecret: fs.Reveal(rcloneClientSecret),
RedirectURL: oauthutil.RedirectLocalhostURL,
}
)
// Register with Fs
func init() {
fs.Register(&fs.Info{
Name: "hubic",
NewFs: NewFs,
Config: func(name string) {
err := oauthutil.Config(name, oauthConfig)
if err != nil {
log.Fatalf("Failed to configure token: %v", err)
}
},
Options: []fs.Option{{
Name: oauthutil.ConfigClientID,
Help: "Hubic Client Id - leave blank normally.",
}, {
Name: oauthutil.ConfigClientSecret,
Help: "Hubic Client Secret - leave blank normally.",
}},
})
}
// credentials is the JSON returned from the Hubic API to read the
// OpenStack credentials
type credentials struct {
Token string `json:"token"` // Openstack token
Endpoint string `json:"endpoint"` // Openstack endpoint
Expires string `json:"expires"` // Expires date - eg "2015-11-09T14:24:56+01:00"
}
// Fs represents a remote hubic
type Fs struct {
fs.Fs // wrapped Fs
client *http.Client // client for oauth api
credentials credentials // returned from the Hubic API
expires time.Time // time credentials expire
}
// Object describes a swift object
type Object struct {
*swift.Object
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.Object.String()
}
// ------------------------------------------------------------
// String converts this Fs to a string
func (f *Fs) String() string {
if f.Fs == nil {
return "Hubic"
}
return fmt.Sprintf("Hubic %s", f.Fs.String())
}
// checkClose is a utility function used to check the return from
// Close in a defer statement.
func checkClose(c io.Closer, err *error) {
cerr := c.Close()
if *err == nil {
*err = cerr
}
}
// getCredentials reads the OpenStack Credentials using the Hubic API
//
// The credentials are read into the Fs
func (f *Fs) getCredentials() (err error) {
req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
if err != nil {
return err
}
req.Header.Add("User-Agent", fs.UserAgent)
resp, err := f.client.Do(req)
if err != nil {
return err
}
defer checkClose(resp.Body, &err)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("Failed to get credentials: %s", resp.Status)
}
decoder := json.NewDecoder(resp.Body)
var result credentials
err = decoder.Decode(&result)
if err != nil {
return err
}
// fs.Debug(f, "Got credentials %+v", result)
if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
return fmt.Errorf("Couldn't read token, result and expired from credentials")
}
f.credentials = result
expires, err := time.Parse(time.RFC3339, result.Expires)
if err != nil {
return err
}
f.expires = expires
fs.Debug(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now()))
return nil
}
// NewFs constructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) {
client, err := oauthutil.NewClient(name, oauthConfig)
if err != nil {
return nil, fmt.Errorf("Failed to configure Hubic: %v", err)
}
f := &Fs{
client: client,
}
// Make the swift Connection
c := &swiftLib.Connection{
Auth: newAuth(f),
UserAgent: fs.UserAgent,
ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
Transport: fs.Config.Transport(),
}
err = c.Authenticate()
if err != nil {
return nil, fmt.Errorf("Error authenticating swift connection: %v", err)
}
// Make inner swift Fs from the connection
swiftFs, err := swift.NewFsWithConnection(name, root, c)
if err != nil {
return nil, err
}
f.Fs = swiftFs
return f, nil
}
// Purge deletes all the files and the container
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
func (f *Fs) Purge() error {
fPurge, ok := f.Fs.(fs.Purger)
if !ok {
return fs.ErrorCantPurge
}
return fPurge.Purge()
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
fCopy, ok := f.Fs.(fs.Copier)
if !ok {
return nil, fs.ErrorCantCopy
}
return fCopy.Copy(src, remote)
}
// UnWrap returns the Fs that this Fs is wrapping
func (f *Fs) UnWrap() fs.Fs {
return f.Fs
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
)

56
hubic/hubic_test.go Normal file
View File

@ -0,0 +1,56 @@
// Test Hubic filesystem interface
//
// Automatically generated - DO NOT EDIT
// Regenerate with: make gen_tests
package hubic_test
import (
"testing"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
"github.com/ncw/rclone/hubic"
)
func init() {
fstests.NilObject = fs.Object((*hubic.Object)(nil))
fstests.RemoteName = "TestHubic:"
}
// Generic tests for the Fs
func TestInit(t *testing.T) { fstests.TestInit(t) }
func TestFsString(t *testing.T) { fstests.TestFsString(t) }
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }

1
make_manual.py
View File

@ -25,6 +25,7 @@ docs = [
"googlecloudstorage.md",
"amazonclouddrive.md",
"onedrive.md",
"hubic.md",
"local.md",
"changelog.md",
"bugs.md",

1
rclone.go
View File

@ -20,6 +20,7 @@ import (
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/hubic"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/onedrive"
_ "github.com/ncw/rclone/s3"

18
swift/swift.go
View File

@ -159,16 +159,13 @@ func swiftConnection(name string) (*swift.Connection, error) {
return c, nil
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) {
// NewFsWithConnection contstructs an Fs from the path, container:path
// and authenticated connection
func NewFsWithConnection(name, root string, c *swift.Connection) (fs.Fs, error) {
container, directory, err := parsePath(root)
if err != nil {
return nil, err
}
c, err := swiftConnection(name)
if err != nil {
return nil, err
}
f := &Fs{
name: name,
c: *c,
@ -196,6 +193,15 @@ func NewFs(name, root string) (fs.Fs, error) {
return f, nil
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (fs.Fs, error) {
c, err := swiftConnection(name)
if err != nil {
return nil, err
}
return NewFsWithConnection(name, root, c)
}
// Return an FsObject from a path
//
// May return nil if an error occurred