Implement Remote Control for rclone #2111
This implements a remote control protocol activated with the --rc flag and a new command `rclone rc` to use that interface. Still to do * docs - need finishing * testss3-about
Nick Craig-Wood
Remus Bunduc
parent
8bb2854fe4
commit
86e5a35491
|
|
@ -18,6 +18,7 @@ docs = [
|
|||
"docs.md",
|
||||
"remote_setup.md",
|
||||
"filtering.md",
|
||||
"rc.md",
|
||||
"overview.md",
|
||||
|
||||
# Keep these alphabetical by full name
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import (
|
|||
_ "github.com/ncw/rclone/cmd/ncdu"
|
||||
_ "github.com/ncw/rclone/cmd/obscure"
|
||||
_ "github.com/ncw/rclone/cmd/purge"
|
||||
_ "github.com/ncw/rclone/cmd/rc"
|
||||
_ "github.com/ncw/rclone/cmd/rcat"
|
||||
_ "github.com/ncw/rclone/cmd/rmdir"
|
||||
_ "github.com/ncw/rclone/cmd/rmdirs"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ import (
|
|||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fspath"
|
||||
fslog "github.com/ncw/rclone/fs/log"
|
||||
"github.com/ncw/rclone/fs/rc"
|
||||
"github.com/ncw/rclone/fs/rc/rcflags"
|
||||
"github.com/ncw/rclone/lib/atexit"
|
||||
)
|
||||
|
||||
|
|
@ -126,6 +128,7 @@ func init() {
|
|||
// Add global flags
|
||||
configflags.AddFlags(pflag.CommandLine)
|
||||
filterflags.AddFlags(pflag.CommandLine)
|
||||
rcflags.AddFlags(pflag.CommandLine)
|
||||
|
||||
Root.Run = runRoot
|
||||
Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
|
||||
|
|
@ -383,6 +386,9 @@ func initConfig() {
|
|||
// Write the args for debug purposes
|
||||
fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
|
||||
|
||||
// Start the remote control if configured
|
||||
rc.Start(&rcflags.Opt)
|
||||
|
||||
// Setup CPU profiling if desired
|
||||
if *cpuProfile != "" {
|
||||
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
package rc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/ncw/rclone/cmd"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/rc"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
noOutput = false
|
||||
url = "http://localhost:5572/"
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefintion)
|
||||
commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.")
|
||||
commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.")
|
||||
}
|
||||
|
||||
var commandDefintion = &cobra.Command{
|
||||
Use: "rc commands parameter",
|
||||
Short: `Run a command against a running rclone.`,
|
||||
Long: `
|
||||
This runs a command against a running rclone. By default it will use
|
||||
that specified in the --rc-addr command.
|
||||
|
||||
Arguments should be passed in as parameter=value.
|
||||
|
||||
The result will be returned as a JSON object by default.
|
||||
|
||||
Use "rclone rc list" to see a list of all possible commands.`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 1E9, command, args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
if len(args) == 0 {
|
||||
return list()
|
||||
}
|
||||
return run(args)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// do a call from (path, in) to (out, err).
|
||||
//
|
||||
// if err is set, out may be a valid error return or it may be nil
|
||||
func doCall(path string, in rc.Params) (out rc.Params, err error) {
|
||||
// Do HTTP request
|
||||
client := fshttp.NewClient(fs.Config)
|
||||
url := url
|
||||
if !strings.HasSuffix(url, "/") {
|
||||
url += "/"
|
||||
}
|
||||
url += path
|
||||
data, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to encode JSON")
|
||||
}
|
||||
resp, err := client.Post(url, "application/json", bytes.NewBuffer(data))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "connection failed")
|
||||
}
|
||||
defer fs.CheckClose(resp.Body, &err)
|
||||
|
||||
// Parse output
|
||||
out = make(rc.Params)
|
||||
err = json.NewDecoder(resp.Body).Decode(&out)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decode JSON")
|
||||
}
|
||||
|
||||
// Check we got 200 OK
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err = errors.Errorf("operation %q failed: %v", path, out["error"])
|
||||
}
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Run the remote control command passed in
|
||||
func run(args []string) (err error) {
|
||||
path := strings.Trim(args[0], "/")
|
||||
|
||||
// parse input
|
||||
in := make(rc.Params)
|
||||
for _, param := range args[1:] {
|
||||
equals := strings.IndexRune(param, '=')
|
||||
if equals < 0 {
|
||||
return errors.Errorf("No '=' found in parameter %q", param)
|
||||
}
|
||||
key, value := param[:equals], param[equals+1:]
|
||||
in[key] = value
|
||||
}
|
||||
|
||||
// Do the call
|
||||
out, callErr := doCall(path, in)
|
||||
|
||||
// Write the JSON blob to stdout if required
|
||||
if out != nil && !noOutput {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(out)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to output JSON")
|
||||
}
|
||||
}
|
||||
|
||||
return callErr
|
||||
}
|
||||
|
||||
// List the available commands to stdout
|
||||
func list() error {
|
||||
list, err := doCall("rc/list", nil)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to list")
|
||||
}
|
||||
commands, ok := list["commands"].([]interface{})
|
||||
if !ok {
|
||||
return errors.New("bad JSON")
|
||||
}
|
||||
for _, command := range commands {
|
||||
info, ok := command.(map[string]interface{})
|
||||
if !ok {
|
||||
return errors.New("bad JSON")
|
||||
}
|
||||
fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"])
|
||||
fmt.Printf("%s\n\n", info["Help"])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -985,6 +985,16 @@ For the filtering options
|
|||
|
||||
See the [filtering section](/filtering/).
|
||||
|
||||
Remote control
|
||||
--------------
|
||||
|
||||
For the remote control options and for instructions on how to remote control rclone
|
||||
|
||||
* `--rc`
|
||||
* and anything starting with `--rc-`
|
||||
|
||||
See [the remote control section](/rc/).
|
||||
|
||||
Logging
|
||||
-------
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,145 @@
|
|||
---
|
||||
title: "Remote Control"
|
||||
description: "Remote controlling rclone"
|
||||
date: "2018-03-05"
|
||||
---
|
||||
|
||||
# Remote controlling rclone #
|
||||
|
||||
If rclone is run with the `--rc` flag then it starts an http server
|
||||
which can be used to remote control rclone.
|
||||
|
||||
FIXME describe other flags
|
||||
|
||||
## Accessing the remote control via the rclone rc command
|
||||
|
||||
Rclone itself implements the remote control protocol in its `rclone
|
||||
rc` command.
|
||||
|
||||
You can use it like this
|
||||
|
||||
|
||||
|
||||
## Accessing the remote control via HTTP
|
||||
|
||||
Rclone implements a simple HTTP based protocol.
|
||||
|
||||
Each endpoint takes an JSON object and returns a JSON object or an
|
||||
error. The JSON objects are essentially a map of string names to
|
||||
values.
|
||||
|
||||
All calls must made using POST.
|
||||
|
||||
The input objects can be supplied using URL parameters, POST
|
||||
parameters or by supplying "Content-Type: application/json" and a JSON
|
||||
blob in the body. There are examples of these below using `curl`.
|
||||
|
||||
The response will be a JSON blob in the body of the response. This is
|
||||
formatted to be reasonably human readable.
|
||||
|
||||
If an error occurs then there will be an HTTP error status (usually
|
||||
400) and the body of the response will contain a JSON encoded error
|
||||
object.
|
||||
|
||||
### Using POST with URL parameters only
|
||||
|
||||
```
|
||||
curl -X POST 'http://localhost:5572/rc/noop/?potato=1&sausage=2'
|
||||
```
|
||||
|
||||
Response
|
||||
|
||||
```
|
||||
{
|
||||
"potato": "1",
|
||||
"sausage": "2"
|
||||
}
|
||||
```
|
||||
|
||||
Here is what an error response looks like:
|
||||
|
||||
```
|
||||
curl -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
|
||||
```
|
||||
|
||||
```
|
||||
{
|
||||
"error": "arbitrary error on input map[potato:1 sausage:2]",
|
||||
"input": {
|
||||
"potato": "1",
|
||||
"sausage": "2"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note that curl doesn't return errors to the shell unless you use the `-f` option
|
||||
|
||||
```
|
||||
$ curl -f -X POST 'http://localhost:5572/rc/error/?potato=1&sausage=2'
|
||||
curl: (22) The requested URL returned error: 400 Bad Request
|
||||
$ echo $?
|
||||
22
|
||||
```
|
||||
|
||||
### Using POST with a form
|
||||
|
||||
```
|
||||
curl --data "potato=1" --data "sausage=2" http://localhost:5572/rc/noop/
|
||||
```
|
||||
|
||||
Response
|
||||
|
||||
```
|
||||
{
|
||||
"potato": "1",
|
||||
"sausage": "2"
|
||||
}
|
||||
```
|
||||
|
||||
Note that you can combine these with URL parameters too with the POST
|
||||
parameters taking precedence.
|
||||
|
||||
```
|
||||
curl --data "potato=1" --data "sausage=2" "http://localhost:5572/rc/noop/?rutabaga=3&sausage=4"
|
||||
```
|
||||
|
||||
Response
|
||||
|
||||
```
|
||||
{
|
||||
"potato": "1",
|
||||
"rutabaga": "3",
|
||||
"sausage": "4"
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
### Using POST with a JSON blob
|
||||
|
||||
```
|
||||
curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' http://localhost:5572/rc/noop/
|
||||
```
|
||||
|
||||
response
|
||||
|
||||
```
|
||||
{
|
||||
"password": "xyz",
|
||||
"username": "xyz"
|
||||
}
|
||||
```
|
||||
|
||||
This can be combined with URL parameters too if required. The JSON
|
||||
blob takes precedence.
|
||||
|
||||
```
|
||||
curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' 'http://localhost:5572/rc/noop/?rutabaga=3&potato=4'
|
||||
```
|
||||
|
||||
```
|
||||
{
|
||||
"potato": 2,
|
||||
"rutabaga": "3",
|
||||
"sausage": 1
|
||||
}
|
||||
```
|
||||
|
|
@ -18,6 +18,7 @@
|
|||
<li><a href="/install/"><i class="fa fa-book"></i> Installation</a></li>
|
||||
<li><a href="/docs/"><i class="fa fa-book"></i> Usage</a></li>
|
||||
<li><a href="/filtering/"><i class="fa fa-book"></i> Filtering</a></li>
|
||||
<li><a href="/rc/"><i class="fa fa-book"></i> Remote Control</a></li>
|
||||
<li><a href="/changelog/"><i class="fa fa-book"></i> Changelog</a></li>
|
||||
<li><a href="/bugs/"><i class="fa fa-book"></i> Bugs</a></li>
|
||||
<li><a href="/faq/"><i class="fa fa-book"></i> FAQ</a></li>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
// Define the internal rc functions
|
||||
|
||||
package rc
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
func init() {
|
||||
Add(Call{
|
||||
Path: "rc/noop",
|
||||
Fn: rcNoop,
|
||||
Title: "Echo the input to the output parameters",
|
||||
Help: `
|
||||
This echoes the input parameters to the output parameters for testing
|
||||
purposes. It can be used to check that rclone is still alive and to
|
||||
check that parameter passing is working properly.`,
|
||||
})
|
||||
Add(Call{
|
||||
Path: "rc/error",
|
||||
Fn: rcError,
|
||||
Title: "This returns an error",
|
||||
Help: `
|
||||
This returns an error with the input as part of its error string.
|
||||
Useful for testing error handling.`,
|
||||
})
|
||||
Add(Call{
|
||||
Path: "rc/list",
|
||||
Fn: rcList,
|
||||
Title: "List all the registered remote control commands",
|
||||
Help: `
|
||||
This lists all the registered remote control commands as a JSON map in
|
||||
the commands response.`,
|
||||
})
|
||||
}
|
||||
|
||||
// Echo the input to the ouput parameters
|
||||
func rcNoop(in Params) (out Params, err error) {
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// Return an error regardless
|
||||
func rcError(in Params) (out Params, err error) {
|
||||
return nil, errors.Errorf("arbitrary error on input %+v", in)
|
||||
}
|
||||
|
||||
// List the registered commands
|
||||
func rcList(in Params) (out Params, err error) {
|
||||
out = make(Params)
|
||||
out["commands"] = registry.list()
|
||||
return out, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
// Package rc implements a remote control server and registry for rclone
|
||||
//
|
||||
// To register your internal calls, call rc.Add(path, function). Your
|
||||
// function should take ane return a Param. It can also return an
|
||||
// error. Use rc.NewError to wrap an existing error along with an
|
||||
// http response type if another response other than 500 internal
|
||||
// error is required on error.
|
||||
package rc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/ncw/rclone/cmd/serve/httplib"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Options contains options for the remote control server
|
||||
type Options struct {
|
||||
HTTPOptions httplib.Options
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
// DefaultOpt is the default values used for Options
|
||||
var DefaultOpt = Options{
|
||||
HTTPOptions: httplib.DefaultOpt,
|
||||
Enabled: false,
|
||||
}
|
||||
|
||||
func init() {
|
||||
DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
|
||||
}
|
||||
|
||||
// Start the remote control server if configured
|
||||
func Start(opt *Options) {
|
||||
if opt.Enabled {
|
||||
s := newServer(opt)
|
||||
go s.serve()
|
||||
}
|
||||
}
|
||||
|
||||
// server contains everything to run the server
|
||||
type server struct {
|
||||
srv *httplib.Server
|
||||
}
|
||||
|
||||
func newServer(opt *Options) *server {
|
||||
// Serve on the DefaultServeMux so can have global registrations appear
|
||||
mux := http.DefaultServeMux
|
||||
s := &server{
|
||||
srv: httplib.NewServer(mux, &opt.HTTPOptions),
|
||||
}
|
||||
mux.HandleFunc("/", s.handler)
|
||||
return s
|
||||
}
|
||||
|
||||
// serve runs the http server - doesn't return
|
||||
func (s *server) serve() {
|
||||
fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
|
||||
s.srv.Serve()
|
||||
}
|
||||
|
||||
// writes JSON in out to w
|
||||
func writeJSON(w http.ResponseWriter, out Params) {
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", "\t")
|
||||
err := enc.Encode(out)
|
||||
if err != nil {
|
||||
// can't return the error at this point
|
||||
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handler reads incoming requests and dispatches them
|
||||
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.Trim(r.URL.Path, "/")
|
||||
in := make(Params)
|
||||
|
||||
writeError := func(err error, status int) {
|
||||
fs.Errorf(nil, "rc: %q: error: %v", path, err)
|
||||
w.WriteHeader(status)
|
||||
writeJSON(w, Params{
|
||||
"error": err.Error(),
|
||||
"input": in,
|
||||
})
|
||||
}
|
||||
|
||||
if r.Method != "POST" {
|
||||
writeError(errors.Errorf("method %q not allowed - POST required", r.Method), http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
// Find the call
|
||||
call := registry.get(path)
|
||||
if call == nil {
|
||||
writeError(errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse the POST and URL parameters into r.Form
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
writeError(errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Read the POST and URL parameters into in
|
||||
for k, vs := range r.Form {
|
||||
if len(vs) > 0 {
|
||||
in[k] = vs[len(vs)-1]
|
||||
}
|
||||
}
|
||||
fs.Debugf(nil, "form = %+v", r.Form)
|
||||
|
||||
// Parse a JSON blob from the input
|
||||
if r.Header.Get("Content-Type") == "application/json" {
|
||||
err := json.NewDecoder(r.Body).Decode(&in)
|
||||
if err != nil {
|
||||
writeError(errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
|
||||
out, err := call.Fn(in)
|
||||
if err != nil {
|
||||
writeError(errors.Wrap(err, "remote control command failed"), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
|
||||
writeJSON(w, out)
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// Package rcflags implements command line flags to set up the remote control
|
||||
package rcflags
|
||||
|
||||
import (
|
||||
"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/rc"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// Options set by command line flags
|
||||
var (
|
||||
Opt = rc.DefaultOpt
|
||||
)
|
||||
|
||||
// AddFlags adds the remote control flags to the flagSet
|
||||
func AddFlags(flagSet *pflag.FlagSet) {
|
||||
flags.BoolVarP(flagSet, &Opt.Enabled, "rc", "", false, "Enable the remote control server.")
|
||||
httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions)
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
// Define the registry
|
||||
|
||||
package rc
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
)
|
||||
|
||||
// Params is the input and output type for the Func
|
||||
type Params map[string]interface{}
|
||||
|
||||
// Func defines a type for a remote control function
|
||||
type Func func(in Params) (out Params, err error)
|
||||
|
||||
// Call defines info about a remote control function and is used in
|
||||
// the Add function to create new entry points.
|
||||
type Call struct {
|
||||
Path string // path to activate this RC
|
||||
Fn Func `json:"-"` // function to call
|
||||
Title string // help for the function
|
||||
Help string // multi-line markdown formatted help
|
||||
}
|
||||
|
||||
// Registry holds the list of all the registered remote control functions
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
call map[string]*Call
|
||||
}
|
||||
|
||||
// NewRegistry makes a new registry for remote control functions
|
||||
func NewRegistry() *Registry {
|
||||
return &Registry{
|
||||
call: make(map[string]*Call),
|
||||
}
|
||||
}
|
||||
|
||||
// Add a call to the registry
|
||||
func (r *Registry) add(call Call) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
call.Path = strings.Trim(call.Path, "/")
|
||||
call.Help = strings.TrimSpace(call.Help)
|
||||
fs.Debugf(nil, "Adding path %q to remote control registry", call.Path)
|
||||
r.call[call.Path] = &call
|
||||
}
|
||||
|
||||
// get a Call from a path or nil
|
||||
func (r *Registry) get(path string) *Call {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.call[path]
|
||||
}
|
||||
|
||||
// get a list of all calls
|
||||
func (r *Registry) list() (out []*Call) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, call := range r.call {
|
||||
out = append(out, call)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The global registry
|
||||
var registry = NewRegistry()
|
||||
|
||||
// Add a function to the global registry
|
||||
func Add(call Call) {
|
||||
registry.add(call)
|
||||
}
|
||||
Loading…
Reference in New Issue