rcd: Add Prometheus metrics support - fixes #3858

Signed-off-by: Gary Kim <gary@garykim.dev>

docs/content/rc.md

@@ -9,6 +9,7 @@ date: "2018-03-05"
 If rclone is run with the `--rc` flag then it starts an http server
 which can be used to remote control rclone.
 
+
 If you just want to run a remote control then see the [rcd command](/commands/rclone_rcd/).
 
 **NB** this is experimental and everything here is subject to change!
@@ -85,6 +86,12 @@ style.
 
 Default Off.
 
+### --rc-enable-metrics
+
+Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`.
+
+Default Off.
+
 ### --rc-web-gui
 
 Set this flag to serve the default web gui on the same port as rclone.

fs/accounting/prometheus.go

@@ -0,0 +1,94 @@
+package accounting
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var namespace = "rclone_"
+
+// RcloneCollector is a Prometheus collector for Rclone
+type RcloneCollector struct {
+	bytesTransferred *prometheus.Desc
+	transferSpeed    *prometheus.Desc
+	numOfErrors      *prometheus.Desc
+	numOfCheckFiles  *prometheus.Desc
+	transferredFiles *prometheus.Desc
+	deletes          *prometheus.Desc
+	fatalError       *prometheus.Desc
+	retryError       *prometheus.Desc
+}
+
+// NewRcloneCollector make a new RcloneCollector
+func NewRcloneCollector() *RcloneCollector {
+	return &RcloneCollector{
+		bytesTransferred: prometheus.NewDesc(namespace+"bytes_transferred_total",
+			"Total transferred bytes since the start of the Rclone process",
+			nil, nil,
+		),
+		transferSpeed: prometheus.NewDesc(namespace+"speed",
+			"Average speed in bytes/sec since the start of the Rclone process",
+			nil, nil,
+		),
+		numOfErrors: prometheus.NewDesc(namespace+"errors_total",
+			"Number of errors thrown",
+			nil, nil,
+		),
+		numOfCheckFiles: prometheus.NewDesc(namespace+"checked_files_total",
+			"Number of checked files",
+			nil, nil,
+		),
+		transferredFiles: prometheus.NewDesc(namespace+"files_transferred_total",
+			"Number of transferred files",
+			nil, nil,
+		),
+		deletes: prometheus.NewDesc(namespace+"files_deleted_total",
+			"Total number of files deleted",
+			nil, nil,
+		),
+		fatalError: prometheus.NewDesc(namespace+"fatal_error",
+			"Whether a fatal error has occurred",
+			nil, nil,
+		),
+		retryError: prometheus.NewDesc(namespace+"retry_error",
+			"Whether there has been an error that will be retried",
+			nil, nil,
+		),
+	}
+}
+
+// Describe is part of the Collector interface: https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector
+func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.bytesTransferred
+	ch <- c.transferSpeed
+	ch <- c.numOfErrors
+	ch <- c.numOfCheckFiles
+	ch <- c.transferredFiles
+	ch <- c.deletes
+	ch <- c.fatalError
+	ch <- c.retryError
+}
+
+// Collect is part of the Collector interface: https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector
+func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
+	s := GlobalStats()
+	s.mu.RLock()
+
+	ch <- prometheus.MustNewConstMetric(c.bytesTransferred, prometheus.CounterValue, float64(s.bytes))
+	ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.Speed())
+	ch <- prometheus.MustNewConstMetric(c.numOfErrors, prometheus.CounterValue, float64(s.errors))
+	ch <- prometheus.MustNewConstMetric(c.numOfCheckFiles, prometheus.CounterValue, float64(s.checks))
+	ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers))
+	ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
+	ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
+	ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))
+
+	s.mu.RUnlock()
+}
+
+// bool2Float is a small function to convert a boolean into a float64 value that can be used for Prometheus
+func bool2Float(e bool) float64 {
+	if e {
+		return 1
+	}
+	return 0
+}

fs/accounting/stats.go

@@ -56,13 +56,7 @@ func NewStats() *StatsInfo {
 func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 	out = make(rc.Params)
 	s.mu.RLock()
-	dt := s.totalDuration()
-	dtSeconds := dt.Seconds()
-	speed := 0.0
-	if dt > 0 {
-		speed = float64(s.bytes) / dtSeconds
-	}
-	out["speed"] = speed
+	out["speed"] = s.Speed()
 	out["bytes"] = s.bytes
 	out["errors"] = s.errors
 	out["fatalError"] = s.fatalError
@@ -70,7 +64,7 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 	out["checks"] = s.checks
 	out["transfers"] = s.transfers
 	out["deletes"] = s.deletes
-	out["elapsedTime"] = dtSeconds
+	out["elapsedTime"] = s.totalDuration().Seconds()
 	s.mu.RUnlock()
 	if !s.checking.empty() {
 		var c []string
@@ -101,6 +95,17 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
 	return out, nil
 }
 
+// Speed returns the average speed of the transfer in bytes/second
+func (s *StatsInfo) Speed() float64 {
+	dt := s.totalDuration()
+	dtSeconds := dt.Seconds()
+	speed := 0.0
+	if dt > 0 {
+		speed = float64(s.bytes) / dtSeconds
+	}
+	return speed
+}
+
 func (s *StatsInfo) transferRemoteStats(name string) rc.Params {
 	s.mu.RLock()
 	defer s.mu.RUnlock()

fs/rc/rc.go

@@ -29,6 +29,7 @@ type Options struct {
 	WebGUINoOpenBrowser      bool   // set to disable auto opening browser
 	WebGUIFetchURL           string // set the default url for fetching webgui
 	AccessControlAllowOrigin string // set the access control for CORS configuration
+	EnableMetrics            bool   // set to disable prometheus metrics on /metrics
 	JobExpireDuration        time.Duration
 	JobExpireInterval        time.Duration
 }

fs/rc/rcflags/rcflags.go

@@ -26,6 +26,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &Opt.WebGUINoOpenBrowser, "rc-web-gui-no-open-browser", "", false, "Don't open the browser automatically")
 	flags.StringVarP(flagSet, &Opt.WebGUIFetchURL, "rc-web-fetch-url", "", "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest", "URL to fetch the releases for webgui.")
 	flags.StringVarP(flagSet, &Opt.AccessControlAllowOrigin, "rc-allow-origin", "", "", "Set the allowed origin for CORS.")
+	flags.BoolVarP(flagSet, &Opt.EnableMetrics, "rc-enable-metrics", "", false, "Enable prometheus metrics on /metrics")
 	flags.DurationVarP(flagSet, &Opt.JobExpireDuration, "rc-job-expire-duration", "", Opt.JobExpireDuration, "expire finished async jobs older than this value")
 	flags.DurationVarP(flagSet, &Opt.JobExpireInterval, "rc-job-expire-interval", "", Opt.JobExpireInterval, "interval to check for expired async jobs")
 	httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions)

fs/rc/rcserver/rcserver.go

@@ -16,9 +16,14 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/skratchdot/open-golang/open"
+
 	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/cmd/serve/httplib/serve"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/list"
@@ -26,9 +31,16 @@ import (
 	"github.com/rclone/rclone/fs/rc/jobs"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/lib/random"
-	"github.com/skratchdot/open-golang/open"
 )
 
+var promHandler http.Handler
+
+func init() {
+	rcloneCollector := accounting.NewRcloneCollector()
+	prometheus.MustRegister(rcloneCollector)
+	promHandler = promhttp.Handler()
+}
+
 // Start the remote control server if configured
 //
 // If the server wasn't configured the *Server returned may be nil
@@ -335,6 +347,9 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 		// Serve /[fs]/remote files
 		s.serveRemote(w, r, match[2], match[1])
 		return
+	case path == "metrics" && s.opt.EnableMetrics:
+		promHandler.ServeHTTP(w, r)
+		return
 	case path == "*" && s.opt.Serve:
 		// Serve /* as the remote listing
 		s.serveRoot(w, r)

fs/rc/rcserver/rcserver_test.go

@@ -12,10 +12,12 @@ import (
 	"testing"
 	"time"
 
-	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fs/rc"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/rc"
 )
 
 const (
@@ -481,6 +483,59 @@ func TestMethods(t *testing.T) {
 	testServer(t, tests, &opt)
 }
 
+func TestMetrics(t *testing.T) {
+	stats := accounting.GlobalStats()
+	tests := makeMetricsTestCases(stats)
+	opt := newTestOpt()
+	opt.EnableMetrics = true
+	testServer(t, tests, &opt)
+
+	// Test changing a couple options
+	stats.Bytes(500)
+	stats.Deletes(30)
+	stats.Errors(2)
+	stats.Bytes(324)
+
+	tests = makeMetricsTestCases(stats)
+	testServer(t, tests, &opt)
+}
+
+func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
+	tests = []testRun{{
+		Name:     "Bytes Transferred Metric",
+		URL:      "/metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
+	}, {
+		Name:     "Checked Files Metric",
+		URL:      "/metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
+	}, {
+		Name:     "Errors Metric",
+		URL:      "/metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
+	}, {
+		Name:     "Deleted Files Metric",
+		URL:      "/metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.Deletes(0))),
+	}, {
+		Name:     "Files Transferred Metric",
+		URL:      "/metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
+	},
+	}
+	return
+}
+
 var matchRemoteDirListing = regexp.MustCompile(`<title>List of all rclone remotes.</title>`)
 
 func TestServingRoot(t *testing.T) {