yves/rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/http: factor directory serving routines into httplib/serve and write tests

Browse Source
This commit is contained in:
Nick Craig-Wood 2018-10-28 14:08:20 +00:00
parent b972dcb0ae
commit 370c218c63
6 changed files with 279 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
cmd/serve/http/http.go
View File

@ -1,8 +1,6 @@
package http
import (
"fmt"
"html/template"
"net/http"
"os"
"path"
@ -12,9 +10,9 @@ import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
"github.com/ncw/rclone/cmd/serve/httplib/serve"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/lib/rest"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
@ -105,62 +103,6 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
}
}
// entry is a directory entry
type entry struct {
remote string
URL string
Leaf string
}
// entries represents a directory
type entries []entry
// addEntry adds an entry to that directory
func (es *entries) addEntry(node interface {
Path() string
Name() string
IsDir() bool
}) {
remote := node.Path()
leaf := node.Name()
urlRemote := leaf
if node.IsDir() {
leaf += "/"
urlRemote += "/"
}
*es = append(*es, entry{remote: remote, URL: rest.URLPathEscape(urlRemote), Leaf: leaf})
}
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
// indexData is used to fill in the indexTemplate
type indexData struct {
Title string
Entries entries
}
// error returns an http.StatusInternalServerError and logs the error
func internalError(what interface{}, w http.ResponseWriter, text string, err error) {
fs.CountError(err)
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
// serveDir serves a directory index at dirRemote
func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
// List the directory
@ -169,7 +111,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
http.Error(w, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
serve.Error(dirRemote, w, "Failed to list directory", err)
return
}
if !node.IsDir() {
@ -179,28 +121,17 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
serve.Error(dirRemote, w, "Failed to list directory", err)
return
}
var out entries
// Make the entries for display
directory := serve.NewDirectory(dirRemote)
for _, node := range dirEntries {
out.addEntry(node)
directory.AddEntry(node.Path(), node.IsDir())
}
// Account the transfer
accounting.Stats.Transferring(dirRemote)
defer accounting.Stats.DoneTransferring(dirRemote, true)
fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr)
err = indexTemplate.Execute(w, indexData{
Entries: out,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
})
if err != nil {
internalError(dirRemote, w, "Failed to render template", err)
return
}
directory.Serve(w, r)
}
// serveFile serves a file object at remote
@ -211,7 +142,7 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
http.Error(w, "File not found", http.StatusNotFound)
return
} else if err != nil {
internalError(remote, w, "Failed to find file", err)
serve.Error(remote, w, "Failed to find file", err)
return
}
if !node.IsFile() {
@ -245,7 +176,7 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
// open the object
in, err := file.Open(os.O_RDONLY)
if err != nil {
internalError(remote, w, "Failed to open file", err)
serve.Error(remote, w, "Failed to open file", err)
return
}
defer func() {

31
cmd/serve/http/http_test.go
View File

@ -7,7 +7,6 @@ import (
"io/ioutil"
"net"
"net/http"
"path"
"strings"
"testing"
"time"
@ -202,36 +201,6 @@ func TestGET(t *testing.T) {
}
}
type mockNode struct {
path string
isdir bool
}
func (n mockNode) Path() string { return n.path }
func (n mockNode) Name() string {
if n.path == "" {
return ""
}
return path.Base(n.path)
}
func (n mockNode) IsDir() bool { return n.isdir }
func TestAddEntry(t *testing.T) {
var es entries
es.addEntry(mockNode{path: "", isdir: true})
es.addEntry(mockNode{path: "dir", isdir: true})
es.addEntry(mockNode{path: "a/b/c/d.txt", isdir: false})
es.addEntry(mockNode{path: "a/b/c/colon:colon.txt", isdir: false})
es.addEntry(mockNode{path: "\"quotes\".txt", isdir: false})
assert.Equal(t, entries{
{remote: "", URL: "/", Leaf: "/"},
{remote: "dir", URL: "dir/", Leaf: "dir/"},
{remote: "a/b/c/d.txt", URL: "d.txt", Leaf: "d.txt"},
{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", Leaf: "colon:colon.txt"},
{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", Leaf: "\"quotes\".txt"},
}, es)
}
func TestFinalise(t *testing.T) {
httpServer.Close()
httpServer.Wait()

102
cmd/serve/httplib/serve/dir.go Normal file
View File

@ -0,0 +1,102 @@
package serve
import (
"fmt"
"html/template"
"net/http"
"net/url"
"path"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/lib/rest"
)
// DirEntry is a directory entry
type DirEntry struct {
remote string
URL string
Leaf string
}
// Directory represents a directory
type Directory struct {
DirRemote string
Title string
Entries []DirEntry
Query string
}
// NewDirectory makes an empty Directory
func NewDirectory(dirRemote string) *Directory {
d := &Directory{
DirRemote: dirRemote,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
}
return d
}
// SetQuery sets the query parameters for each URL
func (d *Directory) SetQuery(queryParams url.Values) *Directory {
d.Query = ""
if len(queryParams) > 0 {
d.Query = "?" + queryParams.Encode()
}
return d
}
// AddEntry adds an entry to that directory
func (d *Directory) AddEntry(remote string, isDir bool) {
leaf := path.Base(remote)
if leaf == "." {
leaf = ""
}
urlRemote := leaf
if isDir {
leaf += "/"
urlRemote += "/"
}
d.Entries = append(d.Entries, DirEntry{
remote: remote,
URL: rest.URLPathEscape(urlRemote) + d.Query,
Leaf: leaf,
})
}
// Error returns an http.StatusInternalServerError and logs the error
func Error(what interface{}, w http.ResponseWriter, text string, err error) {
fs.CountError(err)
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
// Serve serves a directory
func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
// Account the transfer
accounting.Stats.Transferring(d.DirRemote)
defer accounting.Stats.DoneTransferring(d.DirRemote, true)
fs.Infof(d.DirRemote, "%s: Serving directory", r.RemoteAddr)
err := indexTemplate.Execute(w, d)
if err != nil {
Error(d.DirRemote, w, "Failed to render template", err)
return
}
}
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))

88
cmd/serve/httplib/serve/dir_test.go Normal file
View File

@ -0,0 +1,88 @@
package serve
import (
"errors"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNewDirectory(t *testing.T) {
d := NewDirectory("z")
assert.Equal(t, "z", d.DirRemote)
assert.Equal(t, "Directory listing of /z", d.Title)
}
func TestSetQuery(t *testing.T) {
d := NewDirectory("z")
assert.Equal(t, "", d.Query)
d.SetQuery(url.Values{"potato": []string{"42"}})
assert.Equal(t, "?potato=42", d.Query)
d.SetQuery(url.Values{})
assert.Equal(t, "", d.Query)
}
func TestAddEntry(t *testing.T) {
var d = NewDirectory("z")
d.AddEntry("", true)
d.AddEntry("dir", true)
d.AddEntry("a/b/c/d.txt", false)
d.AddEntry("a/b/c/colon:colon.txt", false)
d.AddEntry("\"quotes\".txt", false)
assert.Equal(t, []DirEntry{
{remote: "", URL: "/", Leaf: "/"},
{remote: "dir", URL: "dir/", Leaf: "dir/"},
{remote: "a/b/c/d.txt", URL: "d.txt", Leaf: "d.txt"},
{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", Leaf: "colon:colon.txt"},
{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", Leaf: "\"quotes\".txt"},
}, d.Entries)
// Now test with a query parameter
d = NewDirectory("z").SetQuery(url.Values{"potato": []string{"42"}})
d.AddEntry("file", false)
d.AddEntry("dir", true)
assert.Equal(t, []DirEntry{
{remote: "file", URL: "file?potato=42", Leaf: "file"},
{remote: "dir", URL: "dir/?potato=42", Leaf: "dir/"},
}, d.Entries)
}
func TestError(t *testing.T) {
w := httptest.NewRecorder()
err := errors.New("help")
Error("potato", w, "sausage", err)
resp := w.Result()
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "sausage.\n", string(body))
}
func TestServe(t *testing.T) {
d := NewDirectory("aDirectory")
d.AddEntry("file", false)
d.AddEntry("dir", true)
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://example.com/aDirectory/", nil)
d.Serve(w, r)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Directory listing of /aDirectory</title>
</head>
<body>
<h1>Directory listing of /aDirectory</h1>
<a href="file">file</a><br />
<a href="dir/">dir/</a><br />
</body>
</html>
`, string(body))
}

4
cmd/serve/httplib/serve/serve.go
View File

@ -15,8 +15,12 @@ import (
func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
if r.Method != "HEAD" && r.Method != "GET" {
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
return
}
// Show that we accept ranges
w.Header().Set("Accept-Ranges", "bytes")
// Set content length since we know how long the object is
if o.Size() >= 0 {
w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))

76
cmd/serve/httplib/serve/serve_test.go Normal file
View File

@ -0,0 +1,76 @@
package serve
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/ncw/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
)
func TestObjectBadMethod(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("BADMETHOD", "http://example.com/aFile", nil)
o := mockobject.New("aFile")
Object(w, r, o)
resp := w.Result()
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "Method Not Allowed\n", string(body))
}
func TestObjectHEAD(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil)
o := mockobject.New("aFile").WithContent([]byte("hello"), mockobject.SeekModeNone)
Object(w, r, o)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "5", resp.Header.Get("Content-Length"))
assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "", string(body))
}
func TestObjectGET(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://example.com/aFile", nil)
o := mockobject.New("aFile").WithContent([]byte("hello"), mockobject.SeekModeNone)
Object(w, r, o)
resp := w.Result()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "5", resp.Header.Get("Content-Length"))
assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "hello", string(body))
}
func TestObjectRange(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://example.com/aFile", nil)
r.Header.Add("Range", "bytes=3-5")
o := mockobject.New("aFile").WithContent([]byte("0123456789"), mockobject.SeekModeNone)
Object(w, r, o)
resp := w.Result()
assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
assert.Equal(t, "3", resp.Header.Get("Content-Length"))
assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range"))
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "345", string(body))
}
func TestObjectBadRange(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "http://example.com/aFile", nil)
r.Header.Add("Range", "xxxbytes=3-5")
o := mockobject.New("aFile").WithContent([]byte("0123456789"), mockobject.SeekModeNone)
Object(w, r, o)
resp := w.Result()
assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
assert.Equal(t, "10", resp.Header.Get("Content-Length"))
body, _ := ioutil.ReadAll(resp.Body)
assert.Equal(t, "Bad Request\n", string(body))
}