Implement Backblaze B2 - fixes #224
Nick Craig-Wood
parent
113624691a
commit
33b3eea6ec
|
|
@ -19,6 +19,7 @@ Rclone is a command line program to sync files and directories to and from
|
|||
* Amazon Cloud Drive
|
||||
* Microsoft One Drive
|
||||
* Hubic
|
||||
* Backblaze B2
|
||||
* The local filesystem
|
||||
|
||||
Features
|
||||
|
|
|
|||
|
|
@ -0,0 +1,147 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Error describes a B2 error response
|
||||
type Error struct {
|
||||
Status int `json:"status"` // The numeric HTTP status code. Always matches the status in the HTTP response.
|
||||
Code string `json:"code"` // A single-identifier code that identifies the error.
|
||||
Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
|
||||
}
|
||||
|
||||
// Error statisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
|
||||
}
|
||||
|
||||
// Account describes a B2 account
|
||||
type Account struct {
|
||||
ID string `json:"accountId"` // The identifier for the account.
|
||||
}
|
||||
|
||||
// Bucket describes a B2 bucket
|
||||
type Bucket struct {
|
||||
ID string `json:"bucketId"`
|
||||
AccountID string `json:"accountId"`
|
||||
Name string `json:"bucketName"`
|
||||
Type string `json:"bucketType"`
|
||||
}
|
||||
|
||||
// Timestamp is a UTC time when this file was uploaded. It is a base
|
||||
// 10 number of milliseconds since midnight, January 1, 1970 UTC. This
|
||||
// fits in a 64 bit integer such as the type "long" in the programming
|
||||
// language Java. It is intended to be compatible with Java's time
|
||||
// long. For example, it can be passed directly into the java call
|
||||
// Date.setTime(long time).
|
||||
type Timestamp time.Time
|
||||
|
||||
// MarshalJSON turns a Timestamp into JSON (in UTC)
|
||||
func (t *Timestamp) MarshalJSON() (out []byte, err error) {
|
||||
timestamp := (*time.Time)(t).UTC().UnixNano()
|
||||
return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON turns JSON into a Timestamp
|
||||
func (t *Timestamp) UnmarshalJSON(data []byte) error {
|
||||
timestamp, err := strconv.ParseInt(string(data), 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6))
|
||||
return nil
|
||||
}
|
||||
|
||||
// File is info about a file
|
||||
type File struct {
|
||||
ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
|
||||
Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.
|
||||
Action string `json:"action"` // Either "upload" or "hide". "upload" means a file that was uploaded to B2 Cloud Storage. "hide" means a file version marking the file as hidden, so that it will not show up in b2_list_file_names. The result of b2_list_file_names will contain only "upload". The result of b2_list_file_versions may have both.
|
||||
Size int64 `json:"size"` // The number of bytes in the file.
|
||||
UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded.
|
||||
}
|
||||
|
||||
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
|
||||
type AuthorizeAccountResponse struct {
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
|
||||
APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.
|
||||
DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files.
|
||||
}
|
||||
|
||||
// ListBucketsResponse is as returned from the b2_list_buckets call
|
||||
type ListBucketsResponse struct {
|
||||
Buckets []Bucket `json:"buckets"`
|
||||
}
|
||||
|
||||
// ListFileNamesRequest is as passed to b2_list_file_names or b2_list_file_versions
|
||||
type ListFileNamesRequest struct {
|
||||
BucketID string `json:"bucketId"` // required - The bucket to look for file names in.
|
||||
StartFileName string `json:"startFileName,omitempty"` // optional - The first file name to return. If there is a file with this name, it will be returned in the list. If not, the first file name after this the first one after this name.
|
||||
MaxFileCount int `json:"maxFileCount,omitempty"` // optional - The maximum number of files to return from this call. The default value is 100, and the maximum allowed is 1000.
|
||||
StartFileID string `json:"startFileId,omitempty"` // optional - What to pass in to startFileId for the next search to continue where this one left off.
|
||||
}
|
||||
|
||||
// ListFileNamesResponse is as received from b2_list_file_names or b2_list_file_versions
|
||||
type ListFileNamesResponse struct {
|
||||
Files []File `json:"files"` // An array of objects, each one describing one file.
|
||||
NextFileName *string `json:"nextFileName"` // What to pass in to startFileName for the next search to continue where this one left off, or null if there are no more files.
|
||||
NextFileID *string `json:"nextFileId"` // What to pass in to startFileId for the next search to continue where this one left off, or null if there are no more files.
|
||||
}
|
||||
|
||||
// GetUploadURLRequest is passed to b2_get_upload_url
|
||||
type GetUploadURLRequest struct {
|
||||
BucketID string `json:"bucketId"` // The ID of the bucket that you want to upload to.
|
||||
}
|
||||
|
||||
// GetUploadURLResponse is received from b2_get_upload_url
|
||||
type GetUploadURLResponse struct {
|
||||
BucketID string `json:"bucketId"` // The unique ID of the bucket.
|
||||
UploadURL string `json:"uploadUrl"` // The URL that can be used to upload files to this bucket, see b2_upload_file.
|
||||
AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file.
|
||||
}
|
||||
|
||||
// FileInfo is received from b2_upload_file and b2_get_file_info
|
||||
type FileInfo struct {
|
||||
ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
|
||||
Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.
|
||||
AccountID string `json:"accountId"` // Your account ID.
|
||||
BucketID string `json:"bucketId"` // The bucket that the file is in.
|
||||
Size int64 `json:"contentLength"` // The number of bytes stored in the file.
|
||||
SHA1 string `json:"contentSha1"` // The SHA1 of the bytes stored in the file.
|
||||
ContentType string `json:"contentType"` // The MIME type of the file.
|
||||
Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.
|
||||
}
|
||||
|
||||
// CreateBucketRequest is used to create a bucket
|
||||
type CreateBucketRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Name string `json:"bucketName"`
|
||||
Type string `json:"bucketType"`
|
||||
}
|
||||
|
||||
// DeleteBucketRequest is used to create a bucket
|
||||
type DeleteBucketRequest struct {
|
||||
ID string `json:"bucketId"`
|
||||
AccountID string `json:"accountId"`
|
||||
}
|
||||
|
||||
// DeleteFileRequest is used to delete a file version
|
||||
type DeleteFileRequest struct {
|
||||
ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions.
|
||||
Name string `json:"fileName"` // The name of this file.
|
||||
}
|
||||
|
||||
// HideFileRequest is used to delete a file
|
||||
type HideFileRequest struct {
|
||||
BucketID string `json:"bucketId"` // The bucket containing the file to hide.
|
||||
Name string `json:"fileName"` // The name of the file to hide.
|
||||
}
|
||||
|
||||
// GetFileInfoRequest is used to return a FileInfo struct with b2_get_file_info
|
||||
type GetFileInfoRequest struct {
|
||||
ID string `json:"fileId"` // The ID of the file, as returned by b2_upload_file, b2_list_file_names, or b2_list_file_versions.
|
||||
}
|
||||
|
|
@ -0,0 +1,972 @@
|
|||
// Package b2 provides an interface to the Backblaze B2 object storage system
|
||||
package b2
|
||||
|
||||
// FIXME if b2 could set the mod time then it has everything else to
|
||||
// implement mod times. It is just missing that bit of API.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/b2/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEndpoint = "https://api.backblaze.com"
|
||||
headerPrefix = "x-bz-info-" // lower case as that is what the server returns
|
||||
timeKey = "src_last_modified_millis"
|
||||
timeHeader = headerPrefix + timeKey
|
||||
sha1Header = "X-Bz-Content-Sha1"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.Info{
|
||||
Name: "b2",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "account",
|
||||
Help: "Account ID",
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: "Application Key",
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service - leave blank normally.",
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Fs represents a remote b2 server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
srv *rest.Client // the connection to the b2 server
|
||||
bucket string // the bucket we are working on
|
||||
bucketIDMutex sync.Mutex // mutex to protect _bucketID
|
||||
_bucketID string // the ID of the bucket we are working on
|
||||
root string // the path we are working on if any
|
||||
info api.AuthorizeAccountResponse // result of authorize call
|
||||
uploadMu sync.Mutex // lock for upload variable
|
||||
upload api.GetUploadURLResponse // result of get upload URL call
|
||||
}
|
||||
|
||||
// Object describes a b2 object
|
||||
//
|
||||
// Will definitely have info
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
info api.File // Info from the b2 object if known
|
||||
modTime time.Time // The modified time of the object if known
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
if f.root == "" {
|
||||
return f.bucket
|
||||
}
|
||||
return f.bucket + "/" + f.root
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *Fs) String() string {
|
||||
if f.root == "" {
|
||||
return fmt.Sprintf("B2 bucket %s", f.bucket)
|
||||
}
|
||||
return fmt.Sprintf("B2 bucket %s path %s", f.bucket, f.root)
|
||||
}
|
||||
|
||||
// Pattern to match a b2 path
|
||||
var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
|
||||
|
||||
// parseParse parses a b2 'url'
|
||||
func parsePath(path string) (bucket, directory string, err error) {
|
||||
parts := matcher.FindStringSubmatch(path)
|
||||
if parts == nil {
|
||||
err = fmt.Errorf("Couldn't find bucket in b2 path %q", path)
|
||||
} else {
|
||||
bucket, directory = parts[1], parts[2]
|
||||
directory = strings.Trim(directory, "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// errorHandler parses a non 2xx error response into an error
|
||||
func errorHandler(resp *http.Response) error {
|
||||
// Decode error response
|
||||
errResponse := new(api.Error)
|
||||
err := rest.DecodeJSON(resp, &errResponse)
|
||||
if err != nil {
|
||||
fs.Debug(nil, "Couldn't decode error response: %v", err)
|
||||
}
|
||||
if errResponse.Code == "" {
|
||||
errResponse.Code = "unknown"
|
||||
}
|
||||
if errResponse.Status == 0 {
|
||||
errResponse.Status = resp.StatusCode
|
||||
}
|
||||
if errResponse.Message == "" {
|
||||
errResponse.Message = "Unknown " + resp.Status
|
||||
}
|
||||
return errResponse
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
bucket, directory, err := parsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &Fs{
|
||||
name: name,
|
||||
bucket: bucket,
|
||||
root: directory,
|
||||
}
|
||||
|
||||
account := fs.ConfigFile.MustValue(name, "account")
|
||||
if account == "" {
|
||||
return nil, errors.New("account not found")
|
||||
}
|
||||
key := fs.ConfigFile.MustValue(name, "key")
|
||||
if key == "" {
|
||||
return nil, errors.New("key not found")
|
||||
}
|
||||
endpoint := fs.ConfigFile.MustValue(name, "endpoint", defaultEndpoint)
|
||||
|
||||
f.srv = rest.NewClient(fs.Config.Client()).SetRoot(endpoint + "/b2api/v1").SetErrorHandler(errorHandler)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/b2_authorize_account",
|
||||
UserName: account,
|
||||
Password: key,
|
||||
}
|
||||
_, err = f.srv.CallJSON(&opts, nil, &f.info)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to authenticate: %v", err)
|
||||
}
|
||||
f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
|
||||
|
||||
if f.root != "" {
|
||||
f.root += "/"
|
||||
// Check to see if the (bucket,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
remote := path.Base(directory)
|
||||
f.root = path.Dir(directory)
|
||||
if f.root == "." {
|
||||
f.root = ""
|
||||
} else {
|
||||
f.root += "/"
|
||||
}
|
||||
obj := f.NewFsObject(remote)
|
||||
if obj != nil {
|
||||
return fs.NewLimited(f, obj), nil
|
||||
}
|
||||
f.root = oldRoot
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// getUploadURL returns the UploadURL and the AuthorizationToken
|
||||
func (f *Fs) getUploadURL() (string, string, error) {
|
||||
f.uploadMu.Lock()
|
||||
defer f.uploadMu.Unlock()
|
||||
bucketID, err := f.getBucketID()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if f.upload.UploadURL == "" || f.upload.AuthorizationToken == "" {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_get_upload_url",
|
||||
}
|
||||
var request = api.GetUploadURLRequest{
|
||||
BucketID: bucketID,
|
||||
}
|
||||
_, err := f.srv.CallJSON(&opts, &request, &f.upload)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("Failed to get upload URL: %v", err)
|
||||
}
|
||||
}
|
||||
return f.upload.UploadURL, f.upload.AuthorizationToken, nil
|
||||
}
|
||||
|
||||
// clearUploadURL clears the current UploadURL and the AuthorizationToken
|
||||
func (f *Fs) clearUploadURL() {
|
||||
f.uploadMu.Lock()
|
||||
f.upload = api.GetUploadURLResponse{}
|
||||
defer f.uploadMu.Unlock()
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *Fs) newFsObjectWithInfo(remote string, info *api.File) fs.Object {
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
if info != nil {
|
||||
// Set info but not headers
|
||||
o.info = *info
|
||||
} else {
|
||||
err := o.readMetaData() // reads info and headers, returning an error
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to read metadata: %s", err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// NewFsObject returns an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *Fs) NewFsObject(remote string) fs.Object {
|
||||
return f.newFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(string, *api.File) error
|
||||
|
||||
// list lists the objects into the function supplied from
|
||||
// the bucket and root supplied
|
||||
//
|
||||
// If prefix is set then startFileName is used as a prefix which all
|
||||
// files must have
|
||||
//
|
||||
// If limit is > 0 then it limits to that many files (must be less
|
||||
// than 1000)
|
||||
//
|
||||
// If hidden is set then it will list the hidden (deleted) files too.
|
||||
func (f *Fs) list(prefix string, limit int, hidden bool, fn listFn) error {
|
||||
bucketID, err := f.getBucketID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkSize := 1000
|
||||
if limit > 0 {
|
||||
chunkSize = limit
|
||||
}
|
||||
var request = api.ListFileNamesRequest{
|
||||
BucketID: bucketID,
|
||||
MaxFileCount: chunkSize,
|
||||
}
|
||||
prefix = f.root + prefix
|
||||
if prefix != "" {
|
||||
request.StartFileName = prefix
|
||||
}
|
||||
var response api.ListFileNamesResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_file_names",
|
||||
}
|
||||
if hidden {
|
||||
opts.Path = "/b2_list_file_versions"
|
||||
}
|
||||
for {
|
||||
_, err = f.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range response.Files {
|
||||
file := &response.Files[i]
|
||||
// Finish if file name no longer has prefix
|
||||
if !strings.HasPrefix(file.Name, prefix) {
|
||||
return nil
|
||||
}
|
||||
err = fn(file.Name[len(prefix):], file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// end if no NextFileName
|
||||
if response.NextFileName == nil {
|
||||
break
|
||||
}
|
||||
request.StartFileName = *response.NextFileName
|
||||
if response.NextFileID != nil {
|
||||
request.StartFileID = *response.NextFileID
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List walks the path returning a channel of FsObjects
|
||||
func (f *Fs) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
if f.bucket == "" {
|
||||
// Return no objects at top level list
|
||||
close(out)
|
||||
fs.Stats.Error()
|
||||
fs.ErrorLog(f, "Can't list objects at root - choose a bucket using lsd")
|
||||
} else {
|
||||
// List the objects
|
||||
go func() {
|
||||
defer close(out)
|
||||
err := f.list("", 0, false, func(remote string, object *api.File) error {
|
||||
if o := f.newFsObjectWithInfo(remote, object); o != nil {
|
||||
out <- o
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// listBucketFn is called from listBuckets to handle a bucket
|
||||
type listBucketFn func(*api.Bucket)
|
||||
|
||||
// listBuckets lists the buckets to the function supplied
|
||||
func (f *Fs) listBuckets(fn listBucketFn) error {
|
||||
var account = api.Account{ID: f.info.AccountID}
|
||||
var response api.ListBucketsResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_buckets",
|
||||
}
|
||||
_, err := f.srv.CallJSON(&opts, &account, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range response.Buckets {
|
||||
fn(&response.Buckets[i])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getBucketID finds the ID for the current bucket name
|
||||
func (f *Fs) getBucketID() (bucketID string, err error) {
|
||||
f.bucketIDMutex.Lock()
|
||||
defer f.bucketIDMutex.Unlock()
|
||||
if f._bucketID != "" {
|
||||
return f._bucketID, nil
|
||||
}
|
||||
err = f.listBuckets(func(bucket *api.Bucket) {
|
||||
if bucket.Name == f.bucket {
|
||||
bucketID = bucket.ID
|
||||
}
|
||||
})
|
||||
if bucketID == "" {
|
||||
err = fmt.Errorf("Couldn't find bucket %q", f.bucket)
|
||||
}
|
||||
f._bucketID = bucketID
|
||||
return bucketID, err
|
||||
}
|
||||
|
||||
// setBucketID sets the ID for the current bucket name
|
||||
func (f *Fs) setBucketID(ID string) {
|
||||
f.bucketIDMutex.Lock()
|
||||
f._bucketID = ID
|
||||
f.bucketIDMutex.Unlock()
|
||||
}
|
||||
|
||||
// clearBucketID clears the ID for the current bucket name
|
||||
func (f *Fs) clearBucketID() {
|
||||
f.bucketIDMutex.Lock()
|
||||
f._bucketID = ""
|
||||
f.bucketIDMutex.Unlock()
|
||||
}
|
||||
|
||||
// ListDir lists the buckets
|
||||
func (f *Fs) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
if f.bucket == "" {
|
||||
// List the buckets
|
||||
go func() {
|
||||
defer close(out)
|
||||
err := f.listBuckets(func(bucket *api.Bucket) {
|
||||
out <- &fs.Dir{
|
||||
Name: bucket.Name,
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.ErrorLog(f, "Error listing buckets: %v", err)
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
// List the directories in the path in the bucket
|
||||
go func() {
|
||||
defer close(out)
|
||||
lastDir := ""
|
||||
err := f.list("", 0, false, func(remote string, object *api.File) error {
|
||||
slash := strings.IndexRune(remote, '/')
|
||||
if slash < 0 {
|
||||
return nil
|
||||
}
|
||||
dir := remote[:slash]
|
||||
if dir == lastDir {
|
||||
return nil
|
||||
}
|
||||
out <- &fs.Dir{
|
||||
Name: dir,
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
}
|
||||
lastDir = dir
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.ErrorLog(f, "Couldn't list bucket %q: %s", f.bucket, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Put the object into the bucket
|
||||
//
|
||||
// Copy the reader in to the new object which is returned
|
||||
//
|
||||
// The new object may have been created if an error is returned
|
||||
func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||
// Temporary Object under construction
|
||||
fs := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
return fs, fs.Update(in, modTime, size)
|
||||
}
|
||||
|
||||
// Mkdir creates the bucket if it doesn't exist
|
||||
func (f *Fs) Mkdir() error {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_create_bucket",
|
||||
}
|
||||
var request = api.CreateBucketRequest{
|
||||
AccountID: f.info.AccountID,
|
||||
Name: f.bucket,
|
||||
Type: "allPrivate",
|
||||
}
|
||||
var response api.Bucket
|
||||
_, err := f.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
if apiErr.Code == "duplicate_bucket_name" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Failed to create bucket: %v", err)
|
||||
}
|
||||
f.setBucketID(response.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rmdir deletes the bucket if the fs is at the root
|
||||
//
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir() error {
|
||||
if f.root != "" {
|
||||
return nil
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_delete_bucket",
|
||||
}
|
||||
bucketID, err := f.getBucketID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var request = api.DeleteBucketRequest{
|
||||
ID: bucketID,
|
||||
AccountID: f.info.AccountID,
|
||||
}
|
||||
var response api.Bucket
|
||||
_, err = f.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to delete bucket: %v", err)
|
||||
}
|
||||
f.clearBucketID()
|
||||
f.clearUploadURL()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Precision of the remote
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return fs.ModTimeNotSupported
|
||||
}
|
||||
|
||||
// deleteByID deletes a file version given Name and ID
|
||||
func (f *Fs) deleteByID(ID, Name string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_delete_file_version",
|
||||
}
|
||||
var request = api.DeleteFileRequest{
|
||||
ID: ID,
|
||||
Name: Name,
|
||||
}
|
||||
var response api.File
|
||||
_, err := f.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to delete %q: %v", Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and directories
|
||||
//
|
||||
// Implemented here so we can make sure we delete old versions.
|
||||
func (f *Fs) Purge() error {
|
||||
var errReturn error
|
||||
var checkErrMutex sync.Mutex
|
||||
var checkErr = func(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
checkErrMutex.Lock()
|
||||
defer checkErrMutex.Unlock()
|
||||
fs.Stats.Error()
|
||||
fs.ErrorLog(f, "Purge error: %v", err)
|
||||
if errReturn == nil {
|
||||
errReturn = err
|
||||
}
|
||||
}
|
||||
|
||||
// Delete Config.Transfers in parallel
|
||||
toBeDeleted := make(chan *api.File, fs.Config.Transfers)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for object := range toBeDeleted {
|
||||
checkErr(f.deleteByID(object.ID, object.Name))
|
||||
}
|
||||
}()
|
||||
}
|
||||
checkErr(f.list("", 0, true, func(remote string, object *api.File) error {
|
||||
fs.Debug(remote, "Deleting (id %q)", object.ID)
|
||||
toBeDeleted <- object
|
||||
return nil
|
||||
}))
|
||||
close(toBeDeleted)
|
||||
wg.Wait()
|
||||
|
||||
checkErr(f.Rmdir())
|
||||
return errReturn
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Fs returns the parent Fs
|
||||
func (o *Object) Fs() fs.Fs {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Md5sum returns the Md5sum of an object returning a lowercase hex string
|
||||
func (o *Object) Md5sum() (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (o *Object) Size() int64 {
|
||||
return o.info.Size
|
||||
}
|
||||
|
||||
// readMetaData gets the metadata if it hasn't already been fetched
|
||||
//
|
||||
// it also sets the info
|
||||
func (o *Object) readMetaData() (err error) {
|
||||
if o.info.ID != "" {
|
||||
return nil
|
||||
}
|
||||
err = o.fs.list(o.remote, 1, false, func(remote string, object *api.File) error {
|
||||
if remote == "" {
|
||||
o.info = *object
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if o.info.ID != "" {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("Object %q not found", o.remote)
|
||||
}
|
||||
|
||||
// timeString returns modTime as the number of milliseconds
|
||||
// elapsed since January 1, 1970 UTC as a decimal string.
|
||||
func timeString(modTime time.Time) string {
|
||||
return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
|
||||
}
|
||||
|
||||
// parseTimeString converts a decimal string number of milliseconds
|
||||
// elapsed since January 1, 1970 UTC into a time.Time
|
||||
func parseTimeString(timeString string) (result time.Time, err error) {
|
||||
if timeString == "" {
|
||||
return result, fmt.Errorf("%q not found in metadata", timeKey)
|
||||
}
|
||||
unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC(), nil
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime() (result time.Time) {
|
||||
if !o.modTime.IsZero() {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// Return the current time if can't read metadata
|
||||
result = time.Now()
|
||||
|
||||
// Read metadata (need ID)
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to read metadata: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// Return the UploadTimestamp if can't get file info
|
||||
result = time.Time(o.info.UploadTimestamp)
|
||||
|
||||
// Now read the metadata for the modified time
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_get_file_info",
|
||||
}
|
||||
var request = api.GetFileInfoRequest{
|
||||
ID: o.info.ID,
|
||||
}
|
||||
var response api.FileInfo
|
||||
_, err = o.fs.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to get file info: %v", err)
|
||||
return result
|
||||
}
|
||||
|
||||
// Parse the result
|
||||
timeString := response.Info[timeKey]
|
||||
parsed, err := parseTimeString(timeString)
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err)
|
||||
return result
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(modTime time.Time) {
|
||||
// Not possible with B2
|
||||
}
|
||||
|
||||
// Storable returns if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// openFile represents an Object open for reading
|
||||
type openFile struct {
|
||||
o *Object // Object we are reading for
|
||||
resp *http.Response // response of the GET
|
||||
body io.Reader // reading from here
|
||||
hash hash.Hash // currently accumulating SHA1
|
||||
bytes int64 // number of bytes read on this connection
|
||||
eof bool // whether we have read end of file
|
||||
}
|
||||
|
||||
// newOpenFile wraps an io.ReadCloser and checks the sha1sum
|
||||
func newOpenFile(o *Object, resp *http.Response) *openFile {
|
||||
file := &openFile{
|
||||
o: o,
|
||||
resp: resp,
|
||||
hash: sha1.New(),
|
||||
}
|
||||
file.body = io.TeeReader(resp.Body, file.hash)
|
||||
return file
|
||||
}
|
||||
|
||||
// Read bytes from the object - see io.Reader
|
||||
func (file *openFile) Read(p []byte) (n int, err error) {
|
||||
n, err = file.body.Read(p)
|
||||
file.bytes += int64(n)
|
||||
if err == io.EOF {
|
||||
file.eof = true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Close the object and checks the length and SHA1 if all the object
|
||||
// was read
|
||||
func (file *openFile) Close() (err error) {
|
||||
// Close the body at the end
|
||||
defer fs.CheckClose(file.resp.Body, &err)
|
||||
|
||||
// If not end of file then can't check SHA1
|
||||
if !file.eof {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check to see we read the correct number of bytes
|
||||
if file.o.Size() != file.bytes {
|
||||
return fmt.Errorf("Object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes)
|
||||
}
|
||||
|
||||
// Check the SHA1
|
||||
receivedSHA1 := file.resp.Header.Get(sha1Header)
|
||||
calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil))
|
||||
if receivedSHA1 != calculatedSHA1 {
|
||||
return fmt.Errorf("Object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check it satisfies the interfaces
|
||||
var _ io.ReadCloser = &openFile{}
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open() (in io.ReadCloser, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Absolute: true,
|
||||
Path: o.fs.info.DownloadURL + "/file/" + urlEncode(o.fs.bucket) + "/" + urlEncode(o.fs.root+o.remote),
|
||||
}
|
||||
resp, err := o.fs.srv.Call(&opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to open for download: %v", err)
|
||||
}
|
||||
|
||||
// Parse the time out of the headers if possible
|
||||
timeString := resp.Header.Get(timeHeader)
|
||||
parsed, err := parseTimeString(timeString)
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to parse mod time string %q: %v", timeString, err)
|
||||
} else {
|
||||
o.modTime = parsed
|
||||
}
|
||||
return newOpenFile(o, resp), nil
|
||||
}
|
||||
|
||||
// dontEncode is the characters that do not need percent-encoding
|
||||
//
|
||||
// The characters that do not need percent-encoding are a subset of
|
||||
// the printable ASCII characters: upper-case letters, lower-case
|
||||
// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")",
|
||||
// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must
|
||||
// be replaced with "%" and the two-digit hex value of the byte.
|
||||
const dontEncode = (`abcdefghijklmnopqrstuvwxyz` +
|
||||
`ABCDEFGHIJKLMNOPQRSTUVWXYZ` +
|
||||
`0123456789` +
|
||||
`._-/~!$'()*;=:@`)
|
||||
|
||||
// noNeedToEncode is a bitmap of characters which don't need % encoding
|
||||
var noNeedToEncode [256]bool
|
||||
|
||||
func init() {
|
||||
for _, c := range dontEncode {
|
||||
noNeedToEncode[c] = true
|
||||
}
|
||||
}
|
||||
|
||||
// urlEncode encodes in with % encoding
|
||||
func urlEncode(in string) string {
|
||||
var out bytes.Buffer
|
||||
for i := 0; i < len(in); i++ {
|
||||
c := in[i]
|
||||
if noNeedToEncode[c] {
|
||||
_ = out.WriteByte(c)
|
||||
} else {
|
||||
_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime and size
|
||||
//
|
||||
// The new object may have been created if an error is returned
|
||||
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) {
|
||||
// Open a temp file to copy the input
|
||||
fd, err := ioutil.TempFile("", "rclone-b2-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(fd.Name()) // Delete the file - may not work on Windows
|
||||
defer func() {
|
||||
_ = fd.Close() // Ignore error may have been closed already
|
||||
_ = os.Remove(fd.Name()) // Delete the file - may have been deleted already
|
||||
}()
|
||||
|
||||
// Copy the input while calculating the sha1
|
||||
hash := sha1.New()
|
||||
teed := io.TeeReader(in, hash)
|
||||
n, err := io.Copy(fd, teed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("Read %d bytes expecting %d", n, size)
|
||||
}
|
||||
calculatedSha1 := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
// Rewind the temporary file
|
||||
_, err = fd.Seek(0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get upload URL
|
||||
UploadURL, AuthorizationToken, err := o.fs.getUploadURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Headers for upload file
|
||||
//
|
||||
// Authorization
|
||||
// required
|
||||
// An upload authorization token, from b2_get_upload_url.
|
||||
//
|
||||
// X-Bz-File-Name
|
||||
// required
|
||||
//
|
||||
// The name of the file, in percent-encoded UTF-8. See Files for requirements on file names. See String Encoding.
|
||||
//
|
||||
// Content-Type
|
||||
// required
|
||||
//
|
||||
// The MIME type of the content of the file, which will be returned in
|
||||
// the Content-Type header when downloading the file. Use the
|
||||
// Content-Type b2/x-auto to automatically set the stored Content-Type
|
||||
// post upload. In the case where a file extension is absent or the
|
||||
// lookup fails, the Content-Type is set to application/octet-stream. The
|
||||
// Content-Type mappings can be purused here.
|
||||
//
|
||||
// X-Bz-Content-Sha1
|
||||
// required
|
||||
//
|
||||
// The SHA1 checksum of the content of the file. B2 will check this when
|
||||
// the file is uploaded, to make sure that the file arrived correctly. It
|
||||
// will be returned in the X-Bz-Content-Sha1 header when the file is
|
||||
// downloaded.
|
||||
//
|
||||
// X-Bz-Info-src_last_modified_millis
|
||||
// optional
|
||||
//
|
||||
// If the original source of the file being uploaded has a last modified
|
||||
// time concept, Backblaze recommends using this spelling of one of your
|
||||
// ten X-Bz-Info-* headers (see below). Using a standard spelling allows
|
||||
// different B2 clients and the B2 web user interface to interoperate
|
||||
// correctly. The value should be a base 10 number which represents a UTC
|
||||
// time when the original source file was last modified. It is a base 10
|
||||
// number of milliseconds since midnight, January 1, 1970 UTC. This fits
|
||||
// in a 64 bit integer such as the type "long" in the programming
|
||||
// language Java. It is intended to be compatible with Java's time
|
||||
// long. For example, it can be passed directly into the Java call
|
||||
// Date.setTime(long time).
|
||||
//
|
||||
// X-Bz-Info-*
|
||||
// optional
|
||||
//
|
||||
// Up to 10 of these headers may be present. The * part of the header
|
||||
// name is replace with the name of a custom field in the file
|
||||
// information stored with the file, and the value is an arbitrary UTF-8
|
||||
// string, percent-encoded. The same info headers sent with the upload
|
||||
// will be returned with the download.
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Absolute: true,
|
||||
Path: UploadURL,
|
||||
Body: fd,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Authorization": AuthorizationToken,
|
||||
"X-Bz-File-Name": urlEncode(o.fs.root + o.remote),
|
||||
"Content-Type": fs.MimeType(o),
|
||||
sha1Header: calculatedSha1,
|
||||
timeHeader: timeString(modTime),
|
||||
},
|
||||
ContentLength: &size,
|
||||
}
|
||||
var response api.FileInfo
|
||||
_, err = o.fs.srv.CallJSON(&opts, nil, &response)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to upload: %v", err)
|
||||
}
|
||||
o.info.ID = response.ID
|
||||
o.info.Name = response.Name
|
||||
o.info.Action = "upload"
|
||||
o.info.Size = response.Size
|
||||
o.info.UploadTimestamp = api.Timestamp(time.Now()) // FIXME not quite right
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove() error {
|
||||
bucketID, err := o.fs.getBucketID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_hide_file",
|
||||
}
|
||||
var request = api.HideFileRequest{
|
||||
BucketID: bucketID,
|
||||
Name: o.info.Name,
|
||||
}
|
||||
var response api.File
|
||||
_, err = o.fs.srv.CallJSON(&opts, &request, &response)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to delete file: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Purger = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
)
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
package b2
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fstest"
|
||||
)
|
||||
|
||||
// Test b2 string encoding
|
||||
// https://www.backblaze.com/b2/docs/string_encoding.html
|
||||
|
||||
var encodeTest = []struct {
|
||||
fullyEncoded string
|
||||
minimallyEncoded string
|
||||
plainText string
|
||||
}{
|
||||
{fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "},
|
||||
{fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"},
|
||||
{fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""},
|
||||
{fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"},
|
||||
{fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"},
|
||||
{fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"},
|
||||
{fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"},
|
||||
{fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"},
|
||||
{fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("},
|
||||
{fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"},
|
||||
{fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"},
|
||||
{fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"},
|
||||
{fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","},
|
||||
{fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"},
|
||||
{fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."},
|
||||
{fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"},
|
||||
{fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"},
|
||||
{fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"},
|
||||
{fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"},
|
||||
{fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"},
|
||||
{fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"},
|
||||
{fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"},
|
||||
{fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"},
|
||||
{fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"},
|
||||
{fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"},
|
||||
{fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"},
|
||||
{fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"},
|
||||
{fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"},
|
||||
{fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"},
|
||||
{fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="},
|
||||
{fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"},
|
||||
{fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"},
|
||||
{fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"},
|
||||
{fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"},
|
||||
{fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"},
|
||||
{fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"},
|
||||
{fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"},
|
||||
{fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"},
|
||||
{fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"},
|
||||
{fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"},
|
||||
{fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"},
|
||||
{fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"},
|
||||
{fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"},
|
||||
{fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"},
|
||||
{fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"},
|
||||
{fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"},
|
||||
{fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"},
|
||||
{fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"},
|
||||
{fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"},
|
||||
{fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"},
|
||||
{fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"},
|
||||
{fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"},
|
||||
{fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"},
|
||||
{fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"},
|
||||
{fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"},
|
||||
{fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"},
|
||||
{fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"},
|
||||
{fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"},
|
||||
{fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"},
|
||||
{fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["},
|
||||
{fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"},
|
||||
{fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"},
|
||||
{fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"},
|
||||
{fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"},
|
||||
{fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"},
|
||||
{fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"},
|
||||
{fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"},
|
||||
{fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"},
|
||||
{fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"},
|
||||
{fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"},
|
||||
{fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"},
|
||||
{fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"},
|
||||
{fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"},
|
||||
{fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"},
|
||||
{fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"},
|
||||
{fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"},
|
||||
{fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"},
|
||||
{fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"},
|
||||
{fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"},
|
||||
{fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"},
|
||||
{fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"},
|
||||
{fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"},
|
||||
{fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"},
|
||||
{fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"},
|
||||
{fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"},
|
||||
{fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"},
|
||||
{fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"},
|
||||
{fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"},
|
||||
{fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"},
|
||||
{fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"},
|
||||
{fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"},
|
||||
{fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"},
|
||||
{fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"},
|
||||
{fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"},
|
||||
{fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"},
|
||||
{fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"},
|
||||
{fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"},
|
||||
{fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"},
|
||||
}
|
||||
|
||||
func TestUrlEncode(t *testing.T) {
|
||||
for _, test := range encodeTest {
|
||||
got := urlEncode(test.plainText)
|
||||
if got != test.minimallyEncoded && got != test.fullyEncoded {
|
||||
t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeString(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in time.Time
|
||||
want string
|
||||
}{
|
||||
{fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"},
|
||||
{fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"},
|
||||
{fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"},
|
||||
} {
|
||||
got := timeString(test.in)
|
||||
if test.want != got {
|
||||
t.Logf("%v: want %v got %v", test.in, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestParseTimeString(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want time.Time
|
||||
wantError string
|
||||
}{
|
||||
{"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""},
|
||||
{"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""},
|
||||
{"", time.Time{}, `"src_last_modified_millis" not found in metadata`},
|
||||
{"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`},
|
||||
} {
|
||||
got, err := parseTimeString(test.in)
|
||||
var gotError string
|
||||
if err != nil {
|
||||
gotError = err.Error()
|
||||
}
|
||||
if test.want != got {
|
||||
t.Logf("%v: want %v got %v", test.in, test.want, got)
|
||||
}
|
||||
if test.wantError != gotError {
|
||||
t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
// Test B2 filesystem interface
|
||||
//
|
||||
// Automatically generated - DO NOT EDIT
|
||||
// Regenerate with: make gen_tests
|
||||
package b2_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/ncw/rclone/b2"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
fstests.NilObject = fs.Object((*b2.Object)(nil))
|
||||
fstests.RemoteName = "TestB2:"
|
||||
}
|
||||
|
||||
// Generic tests for the Fs
|
||||
func TestInit(t *testing.T) { fstests.TestInit(t) }
|
||||
func TestFsString(t *testing.T) { fstests.TestFsString(t) }
|
||||
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
|
||||
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
|
||||
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
|
||||
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
|
||||
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
|
||||
func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) }
|
||||
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
|
||||
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
|
||||
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
|
||||
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
|
||||
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
|
||||
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
|
||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
|
||||
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
|
||||
func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) }
|
||||
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
|
||||
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
|
||||
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
|
||||
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
|
||||
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
|
||||
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
|
||||
func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) }
|
||||
func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) }
|
||||
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
|
||||
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
|
||||
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }
|
||||
|
|
@ -21,6 +21,7 @@ Rclone is a command line program to sync files and directories to and from
|
|||
* Amazon Cloud Drive
|
||||
* Microsoft One Drive
|
||||
* Hubic
|
||||
* Backblaze B2
|
||||
* The local filesystem
|
||||
|
||||
Features
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
---
|
||||
title: "B2"
|
||||
description: "Backblaze B2"
|
||||
date: "2015-12-29"
|
||||
---
|
||||
|
||||
<i class="fa fa-fire"></i>Backblaze B2
|
||||
----------------------------------------
|
||||
|
||||
B2 is [Backblaze's cloud storage system](https://www.backblaze.com/b2/).
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
command.) You may put subdirectories in too, eg `remote:bucket/path/to/dir`.
|
||||
|
||||
Here is an example of making a b2 configuration. First run
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process. You will
|
||||
need your account number (a short hex number) and key (a long hex
|
||||
number) which you can get from the b2 control panel.
|
||||
|
||||
```
|
||||
No remotes found - make a new one
|
||||
n) New remote
|
||||
q) Quit config
|
||||
n/q> n
|
||||
name> remote
|
||||
What type of source is it?
|
||||
Choose a number from below
|
||||
1) amazon cloud drive
|
||||
2) b2
|
||||
3) drive
|
||||
4) dropbox
|
||||
5) google cloud storage
|
||||
6) swift
|
||||
7) hubic
|
||||
8) local
|
||||
9) onedrive
|
||||
10) s3
|
||||
type> 2
|
||||
Account ID
|
||||
account> 123456789abc
|
||||
Application Key
|
||||
key> 0123456789abcdef0123456789abcdef0123456789
|
||||
Endpoint for the service - leave blank normally.
|
||||
endpoint>
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
account = 123456789abc
|
||||
key = 0123456789abcdef0123456789abcdef0123456789
|
||||
endpoint =
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This remote is called `remote` and can now be used like this
|
||||
|
||||
See all buckets
|
||||
|
||||
rclone lsd remote:
|
||||
|
||||
Make a new bucket
|
||||
|
||||
rclone mkdir remote:bucket
|
||||
|
||||
List the contents of a bucket
|
||||
|
||||
rclone ls remote:bucket
|
||||
|
||||
Sync `/home/local/directory` to the remote bucket, deleting any
|
||||
excess files in the bucket.
|
||||
|
||||
rclone sync /home/local/directory remote:bucket
|
||||
|
||||
### Modified time ###
|
||||
|
||||
The modified time is stored as metadata on the object as
|
||||
`X-Bz-Info-src_last_modified_millis` as milliseconds since 1970-01-01
|
||||
in the Backblaze standard. Other tools should be able to use this as
|
||||
a modified time.
|
||||
|
||||
Modified times are set on upload, read on download and shown in
|
||||
listings. They are not used in syncing as unfortunately B2 doesn't
|
||||
have an API method to set them independently of doing an upload.
|
||||
|
||||
### SHA1 checksums ###
|
||||
|
||||
The SHA1 checksums of the files are checked on upload and download,
|
||||
but they aren't used for sync as they aren't compatible with the MD5
|
||||
checksums rclone normally uses.
|
||||
|
||||
### Versions ###
|
||||
|
||||
When rclone uploads a new version of a file it creates a [new version
|
||||
of it](https://www.backblaze.com/b2/docs/file_versions.html).
|
||||
Likewise when you delete a file, the old version will still be
|
||||
available.
|
||||
|
||||
The old versions of files are visible in the B2 web interface, but not
|
||||
via rclone yet.
|
||||
|
||||
Rclone doesn't provide any way of managing old versions (downloading
|
||||
them or deleting them) at the moment. When you `purge` a bucket, all
|
||||
the old versions will be deleted.
|
||||
|
||||
### Bugs ###
|
||||
|
||||
Note that when uploading a file, rclone has to make a temporary copy
|
||||
of it in your temp filing system. This is due to a weakness in the B2
|
||||
API which I'm hoping will be addressed soon.
|
||||
|
||||
### API ###
|
||||
|
||||
Here are [some notes I made on the backblaze
|
||||
API](https://gist.github.com/ncw/166dabf352b399f1cc1c) while
|
||||
integrating it with rclone which detail the changes I'd like to see.
|
||||
With a couple of small tweaks Backblaze could enable rclone to not
|
||||
make a temporary copy of all files and fully support modification
|
||||
times.
|
||||
|
|
@ -25,6 +25,7 @@ Here is an overview of the major features of each cloud storage system.
|
|||
| Amazon Cloud Drive | Yes | No | Yes | No |
|
||||
| Microsoft One Drive | No | Yes | Yes | No |
|
||||
| Hubic | Yes | Yes | No | No |
|
||||
| Backblaze B2 | No | Partial | No | No |
|
||||
| The local filesystem | Yes | Yes | Depends | No |
|
||||
|
||||
### MD5SUM ###
|
||||
|
|
@ -44,6 +45,9 @@ default, though the MD5SUM can be checked with the `--checksum` flag.
|
|||
All cloud storage systems support some kind of date on the object and
|
||||
these will be set when transferring from the cloud storage system.
|
||||
|
||||
Backblaze B2 preserves file modification times on files uploaded and
|
||||
downloaded, but doesn't use them to decide which objects to sync.
|
||||
|
||||
### Case Insensitive ###
|
||||
|
||||
If a cloud storage systems is case sensitive then it is possible to
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@
|
|||
<li><a href="/amazonclouddrive/"><i class="fa fa-amazon"></i> Amazon Cloud Drive</a></li>
|
||||
<li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft One Drive</a></li>
|
||||
<li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
|
||||
<li><a href="/b2/"><i class="fa fa-fire"></i> Backblaze B2</a></li>
|
||||
<li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
|
||||
// Active file systems
|
||||
_ "github.com/ncw/rclone/amazonclouddrive"
|
||||
_ "github.com/ncw/rclone/b2"
|
||||
_ "github.com/ncw/rclone/drive"
|
||||
_ "github.com/ncw/rclone/dropbox"
|
||||
_ "github.com/ncw/rclone/googlecloudstorage"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ var (
|
|||
"TestAmazonCloudDrive:",
|
||||
"TestOneDrive:",
|
||||
"TestHubic:",
|
||||
"TestB2:",
|
||||
}
|
||||
binary = "fs.test"
|
||||
// Flags
|
||||
|
|
|
|||
|
|
@ -133,5 +133,6 @@ func main() {
|
|||
generateTestProgram(t, fns, "AmazonCloudDrive")
|
||||
generateTestProgram(t, fns, "OneDrive")
|
||||
generateTestProgram(t, fns, "Hubic")
|
||||
generateTestProgram(t, fns, "B2")
|
||||
log.Printf("Done")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ docs = [
|
|||
"amazonclouddrive.md",
|
||||
"onedrive.md",
|
||||
"hubic.md",
|
||||
"b2.md",
|
||||
"local.md",
|
||||
"changelog.md",
|
||||
"bugs.md",
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
|||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
srv: rest.NewClient(oAuthClient, rootURL),
|
||||
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
}
|
||||
f.srv.SetErrorHandler(errorHandler)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
"github.com/ncw/rclone/fs"
|
||||
// Active file systems
|
||||
_ "github.com/ncw/rclone/amazonclouddrive"
|
||||
_ "github.com/ncw/rclone/b2"
|
||||
_ "github.com/ncw/rclone/drive"
|
||||
_ "github.com/ncw/rclone/dropbox"
|
||||
_ "github.com/ncw/rclone/googlecloudstorage"
|
||||
|
|
|
|||
61
rest/rest.go
61
rest/rest.go
|
|
@ -6,6 +6,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
|
|
@ -16,27 +17,48 @@ type Client struct {
|
|||
c *http.Client
|
||||
rootURL string
|
||||
errorHandler func(resp *http.Response) error
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
// NewClient takes an oauth http.Client and makes a new api instance
|
||||
func NewClient(c *http.Client, rootURL string) *Client {
|
||||
return &Client{
|
||||
func NewClient(c *http.Client) *Client {
|
||||
api := &Client{
|
||||
c: c,
|
||||
rootURL: rootURL,
|
||||
errorHandler: defaultErrorHandler,
|
||||
headers: make(map[string]string),
|
||||
}
|
||||
api.SetHeader("User-Agent", fs.UserAgent)
|
||||
return api
|
||||
}
|
||||
|
||||
// defaultErrorHandler doesn't attempt to parse the http body
|
||||
// defaultErrorHandler doesn't attempt to parse the http body, just
|
||||
// returns it in the error message
|
||||
func defaultErrorHandler(resp *http.Response) (err error) {
|
||||
defer fs.CheckClose(resp.Body, &err)
|
||||
return fmt.Errorf("HTTP error %v (%v) returned", resp.StatusCode, resp.Status)
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body)
|
||||
}
|
||||
|
||||
// SetErrorHandler sets the handler to decode an error response when
|
||||
// the HTTP status code is not 2xx. The handler should close resp.Body.
|
||||
func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) {
|
||||
func (api *Client) SetErrorHandler(fn func(resp *http.Response) error) *Client {
|
||||
api.errorHandler = fn
|
||||
return api
|
||||
}
|
||||
|
||||
// SetRoot sets the default root URL
|
||||
func (api *Client) SetRoot(RootURL string) *Client {
|
||||
api.rootURL = RootURL
|
||||
return api
|
||||
}
|
||||
|
||||
// SetHeader sets a header for all requests
|
||||
func (api *Client) SetHeader(key, value string) *Client {
|
||||
api.headers[key] = value
|
||||
return api
|
||||
}
|
||||
|
||||
// Opts contains parameters for Call, CallJSON etc
|
||||
|
|
@ -50,6 +72,8 @@ type Opts struct {
|
|||
ContentLength *int64
|
||||
ContentRange string
|
||||
ExtraHeaders map[string]string
|
||||
UserName string // username for Basic Auth
|
||||
Password string // password for Basic Auth
|
||||
}
|
||||
|
||||
// DecodeJSON decodes resp.Body into result
|
||||
|
|
@ -72,27 +96,42 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
|
|||
if opts.Absolute {
|
||||
url = opts.Path
|
||||
} else {
|
||||
if api.rootURL == "" {
|
||||
return nil, fmt.Errorf("RootURL not set")
|
||||
}
|
||||
url = api.rootURL + opts.Path
|
||||
}
|
||||
req, err := http.NewRequest(opts.Method, url, opts.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
headers := make(map[string]string)
|
||||
// Set default headers
|
||||
for k, v := range api.headers {
|
||||
headers[k] = v
|
||||
}
|
||||
if opts.ContentType != "" {
|
||||
req.Header.Add("Content-Type", opts.ContentType)
|
||||
headers["Content-Type"] = opts.ContentType
|
||||
}
|
||||
if opts.ContentLength != nil {
|
||||
req.ContentLength = *opts.ContentLength
|
||||
}
|
||||
if opts.ContentRange != "" {
|
||||
req.Header.Add("Content-Range", opts.ContentRange)
|
||||
headers["Content-Range"] = opts.ContentRange
|
||||
}
|
||||
// Set any extra headers
|
||||
if opts.ExtraHeaders != nil {
|
||||
for k, v := range opts.ExtraHeaders {
|
||||
req.Header.Add(k, v)
|
||||
headers[k] = v
|
||||
}
|
||||
}
|
||||
req.Header.Add("User-Agent", fs.UserAgent)
|
||||
// Now set the headers
|
||||
for k, v := range headers {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
if opts.UserName != "" || opts.Password != "" {
|
||||
req.SetBasicAuth(opts.UserName, opts.Password)
|
||||
}
|
||||
resp, err = api.c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -127,7 +166,7 @@ func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{
|
|||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
if opts.NoResponse {
|
||||
if response == nil || opts.NoResponse {
|
||||
return resp, nil
|
||||
}
|
||||
err = DecodeJSON(resp, response)
|
||||
|
|
|
|||
Loading…
Reference in New Issue