yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add context to all http.NewRequest #3257 

When we drop support for go1.12 we can use http.NewRequestWithContext
s3-about
Nick Craig-Wood 2019-09-04 20:21:10 +01:00
parent 58a531a203
commit 66c23723e3
13 changed files with 70 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
backend/drive/drive.go
View File

@ -1734,7 +1734,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
}
} else {
// Upload the file in chunks
info, err = f.Upload(in, size, srcMimeType, "", remote, createInfo)
info, err = f.Upload(ctx, in, size, srcMimeType, "", remote, createInfo)
if err != nil {
return nil, err
}
@ -2499,7 +2499,7 @@ func (o *baseObject) Storable() bool {
// httpResponse gets an http.Response object for the object
// using the url and method passed in
func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
if url == "" {
return nil, nil, errors.New("forbidden to download - check sharing permission")
}
@ -2507,6 +2507,7 @@ func (o *baseObject) httpResponse(url, method string, options []fs.OpenOption) (
if err != nil {
return req, nil, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
fs.OpenOptionAddHTTPHeaders(req.Header, options)
if o.bytes == 0 {
// Don't supply range requests for 0 length objects as they always fail
@ -2577,8 +2578,8 @@ func isGoogleError(err error, what string) bool {
}
// open a url for reading
func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
_, res, err := o.httpResponse(url, "GET", options)
func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOption) (in io.ReadCloser, err error) {
_, res, err := o.httpResponse(ctx, url, "GET", options)
if err != nil {
if isGoogleError(err, "cannotDownloadAbusiveFile") {
if o.fs.opt.AcknowledgeAbuse {
@ -2589,7 +2590,7 @@ func (o *baseObject) open(url string, options ...fs.OpenOption) (in io.ReadClose
url += "?"
}
url += "acknowledgeAbuse=true"
_, res, err = o.httpResponse(url, "GET", options)
_, res, err = o.httpResponse(ctx, url, "GET", options)
} else {
err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file")
}
@ -2618,7 +2619,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
o.v2Download = false
}
}
return o.baseObject.open(o.url, options...)
return o.baseObject.open(ctx, o.url, options...)
}
func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
// Update the size with what we are reading as it can change from
@ -2643,7 +2644,7 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in
if offset != 0 {
return nil, errors.New("partial downloads are not supported while exporting Google Documents")
}
in, err = o.baseObject.open(o.url, options...)
in, err = o.baseObject.open(ctx, o.url, options...)
if in != nil {
in = &openDocumentFile{o: o, in: in}
}
@ -2678,7 +2679,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
return ioutil.NopCloser(bytes.NewReader(data)), nil
}
func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io.Reader,
func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
src fs.ObjectInfo) (info *drive.File, err error) {
// Make the API request to upload metadata and file data.
size := src.Size()
@ -2696,7 +2697,7 @@ func (o *baseObject) update(updateInfo *drive.File, uploadMimeType string, in io
return
}
// Upload the file in chunks
return o.fs.Upload(in, size, uploadMimeType, o.id, o.remote, updateInfo)
return o.fs.Upload(ctx, in, size, uploadMimeType, o.id, o.remote, updateInfo)
}
// Update the already existing object
@ -2710,7 +2711,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
MimeType: srcMimeType,
ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
}
info, err := o.baseObject.update(updateInfo, srcMimeType, in, src)
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
if err != nil {
return err
}
@ -2747,7 +2748,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
}
updateInfo.MimeType = importMimeType
info, err := o.baseObject.update(updateInfo, srcMimeType, in, src)
info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
if err != nil {
return err
}

21
backend/drive/upload.go
View File

@ -11,6 +11,7 @@
package drive
import (
"context"
"encoding/json"
"fmt"
"io"
@ -50,7 +51,7 @@ type resumableUpload struct {
}
// Upload the io.Reader in of size bytes with contentType and info
func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
params := url.Values{
"alt": {"json"},
"uploadType": {"resumable"},
@ -81,6 +82,7 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
if err != nil {
return false, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
googleapi.Expand(req.URL, map[string]string{
"fileId": fileID,
})
@ -106,12 +108,13 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
MediaType: contentType,
ContentLength: size,
}
return rx.Upload()
return rx.Upload(ctx)
}
// Make an http.Request for the range passed in
func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request {
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
req, _ := http.NewRequest("POST", rx.URI, body)
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.ContentLength = reqSize
if reqSize != 0 {
req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
@ -129,8 +132,8 @@ var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
// Query drive for the amount transferred so far
//
// If error is nil, then start should be valid
func (rx *resumableUpload) transferStatus() (start int64, err error) {
req := rx.makeRequest(0, nil, 0)
func (rx *resumableUpload) transferStatus(ctx context.Context) (start int64, err error) {
req := rx.makeRequest(ctx, 0, nil, 0)
res, err := rx.f.client.Do(req)
if err != nil {
return 0, err
@ -157,9 +160,9 @@ func (rx *resumableUpload) transferStatus() (start int64, err error) {
}
// Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
_, _ = chunk.Seek(0, io.SeekStart)
req := rx.makeRequest(start, chunk, chunkSize)
req := rx.makeRequest(ctx, start, chunk, chunkSize)
res, err := rx.f.client.Do(req)
if err != nil {
return 599, err
@ -192,7 +195,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
// Upload uploads the chunks from the input
// It retries each chunk using the pacer and --low-level-retries
func (rx *resumableUpload) Upload() (*drive.File, error) {
func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
start := int64(0)
var StatusCode int
var err error
@ -207,7 +210,7 @@ func (rx *resumableUpload) Upload() (*drive.File, error) {
// Transfer the chunk
err = rx.f.pacer.Call(func() (bool, error) {
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
StatusCode, err = rx.transferChunk(start, chunk, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
again, err := shouldRetry(err)
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
again = false

1
backend/googlecloudstorage/googlecloudstorage.go
View File

@ -1007,6 +1007,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
if err != nil {
return nil, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
fs.FixRangeOption(options, o.bytes)
fs.OpenOptionAddHTTPHeaders(req.Header, options)
var res *http.Response

15
backend/http/http.go
View File

@ -124,6 +124,7 @@ func statusError(res *http.Response, err error) error {
// NewFs creates a new Fs object from the name and root. It connects to
// the host specified in the config file.
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
ctx := context.TODO()
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@ -162,6 +163,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// check to see if points to a file
req, err := http.NewRequest("HEAD", u.String(), nil)
if err == nil {
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
addHeaders(req, opt)
res, err := noRedir.Do(req)
err = statusError(res, err)
@ -237,7 +239,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
fs: f,
remote: remote,
}
err := o.stat()
err := o.stat(ctx)
if err != nil {
return nil, err
}
@ -355,7 +357,7 @@ func (f *Fs) addHeaders(req *http.Request) {
}
// Read the directory passed in
func (f *Fs) readDir(dir string) (names []string, err error) {
func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error) {
URL := f.url(dir)
u, err := url.Parse(URL)
if err != nil {
@ -369,6 +371,7 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
if err != nil {
return nil, errors.Wrap(err, "readDir failed")
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
f.addHeaders(req)
res, err := f.httpClient.Do(req)
if err == nil {
@ -408,7 +411,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if !strings.HasSuffix(dir, "/") && dir != "" {
dir += "/"
}
names, err := f.readDir(dir)
names, err := f.readDir(ctx, dir)
if err != nil {
return nil, errors.Wrapf(err, "error listing %q", dir)
}
@ -424,7 +427,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
fs: f,
remote: remote,
}
switch err = file.stat(); err {
switch err = file.stat(ctx); err {
case nil:
entries = append(entries, file)
case fs.ErrorNotAFile:
@ -492,12 +495,13 @@ func (o *Object) url() string {
}
// stat updates the info field in the Object
func (o *Object) stat() error {
func (o *Object) stat(ctx context.Context) error {
url := o.url()
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return errors.Wrap(err, "stat failed")
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
o.fs.addHeaders(req)
res, err := o.fs.httpClient.Do(req)
if err == nil && res.StatusCode == http.StatusNotFound {
@ -546,6 +550,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
if err != nil {
return nil, errors.Wrap(err, "Open failed")
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
// Add optional headers
for k, v := range fs.OpenOptionHeaders(options) {

3
backend/hubic/auth.go
View File

@ -1,6 +1,7 @@
package hubic
import (
"context"
"net/http"
"time"
@ -26,7 +27,7 @@ func newAuth(f *Fs) *auth {
func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {
const retries = 10
for try := 1; try <= retries; try++ {
err = a.f.getCredentials()
err = a.f.getCredentials(context.TODO())
if err == nil {
break
}

4
backend/hubic/hubic.go
View File

@ -7,6 +7,7 @@ package hubic
// to be revisted after some actual experience.
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
@ -115,11 +116,12 @@ func (f *Fs) String() string {
// getCredentials reads the OpenStack Credentials using the Hubic API
//
// The credentials are read into the Fs
func (f *Fs) getCredentials() (err error) {
func (f *Fs) getCredentials(ctx context.Context) (err error) {
req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
if err != nil {
return err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
resp, err := f.client.Do(req)
if err != nil {
return err

6
backend/putio/fs.go
View File

@ -289,6 +289,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
if err != nil {
return false, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.Header.Set("tus-resumable", "1.0.0")
req.Header.Set("upload-length", strconv.FormatInt(size, 10))
b64name := base64.StdEncoding.EncodeToString([]byte(name))
@ -354,7 +355,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
func (f *Fs) transferChunk(ctx context.Context, location string, start int64, chunk io.ReadSeeker, chunkSize int64) (fileID int64, err error) {
// defer log.Trace(f, "location=%v, start=%v, chunkSize=%v", location, start, chunkSize)("fileID=%v, err=%v", fileID, &err)
_, _ = chunk.Seek(0, io.SeekStart)
req, err := f.makeUploadPatchRequest(location, chunk, start, chunkSize)
req, err := f.makeUploadPatchRequest(ctx, location, chunk, start, chunkSize)
if err != nil {
return 0, err
}
@ -379,11 +380,12 @@ func (f *Fs) transferChunk(ctx context.Context, location string, start int64, ch
return fileID, nil
}
func (f *Fs) makeUploadPatchRequest(location string, in io.Reader, offset, length int64) (*http.Request, error) {
func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) {
req, err := http.NewRequest("PATCH", location, in)
if err != nil {
return nil, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.Header.Set("tus-resumable", "1.0.0")
req.Header.Set("upload-offset", strconv.FormatInt(offset, 10))
req.Header.Set("content-length", strconv.FormatInt(length, 10))

6
backend/putio/object.go
View File

@ -223,7 +223,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var resp *http.Response
headers := fs.OpenOptionHeaders(options)
err = o.fs.pacer.Call(func() (bool, error) {
req, _ := http.NewRequest(http.MethodGet, storageURL, nil)
req, err := http.NewRequest(http.MethodGet, storageURL, nil)
if err != nil {
return shouldRetry(err)
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.Header.Set("User-Agent", o.fs.client.UserAgent)
// merge headers with extra headers

2
backend/s3/s3.go
View File

@ -1942,7 +1942,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil {
return errors.Wrap(err, "s3 upload: new request")
}
httpReq = httpReq.WithContext(ctx)
httpReq = httpReq.WithContext(ctx) // go1.13 can use NewRequestWithContext
// set the headers we signed and the length
httpReq.Header = headers

8
backend/webdav/odrvcookie/fetch.go
View File

@ -3,6 +3,7 @@ package odrvcookie
import (
"bytes"
"context"
"encoding/xml"
"html/template"
"net/http"
@ -91,8 +92,8 @@ func New(pUser, pPass, pEndpoint string) CookieAuth {
// Cookies creates a CookieResponse. It fetches the auth token and then
// retrieves the Cookies
func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
tokenResp, err := ca.getSPToken()
func (ca *CookieAuth) Cookies(ctx context.Context) (*CookieResponse, error) {
tokenResp, err := ca.getSPToken(ctx)
if err != nil {
return nil, err
}
@ -140,7 +141,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
return &cookieResponse, nil
}
func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) {
func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SuccessResponse, err error) {
reqData := map[string]interface{}{
"Username": ca.user,
"Password": ca.pass,
@ -160,6 +161,7 @@ func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) {
if err != nil {
return nil, err
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
client := fshttp.NewClient(fs.Config)
resp, err := client.Do(req)

8
backend/webdav/webdav.go
View File

@ -353,7 +353,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
}
f.srv.SetErrorHandler(errorHandler)
err = f.setQuirks(opt.Vendor)
err = f.setQuirks(ctx, opt.Vendor)
if err != nil {
return nil, err
}
@ -424,7 +424,7 @@ func (f *Fs) fetchAndSetBearerToken() error {
}
// setQuirks adjusts the Fs for the vendor passed in
func (f *Fs) setQuirks(vendor string) error {
func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
switch vendor {
case "owncloud":
f.canStream = true
@ -440,13 +440,13 @@ func (f *Fs) setQuirks(vendor string) error {
// They have to be set instead of BasicAuth
f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies
spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL)
spCookies, err := spCk.Cookies()
spCookies, err := spCk.Cookies(ctx)
if err != nil {
return err
}
odrvcookie.NewRenew(12*time.Hour, func() {
spCookies, err := spCk.Cookies()
spCookies, err := spCk.Cookies(ctx)
if err != nil {
fs.Errorf("could not renew cookies: %s", err.Error())
return

16
cmd/rc/rc.go
View File

@ -71,11 +71,12 @@ Use "rclone rc" to see a list of all possible commands.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 1e9, command, args)
cmd.Run(false, false, command, func() error {
ctx := context.Background()
parseFlags()
if len(args) == 0 {
return list()
return list(ctx)
}
return run(args)
return run(ctx, args)
})
},
}
@ -110,7 +111,7 @@ func setAlternateFlag(flagName string, output *string) {
// do a call from (path, in) to (out, err).
//
// if err is set, out may be a valid error return or it may be nil
func doCall(path string, in rc.Params) (out rc.Params, err error) {
func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err error) {
// If loopback set, short circuit HTTP request
if loopback {
call := rc.Calls.Get(path)
@ -141,6 +142,7 @@ func doCall(path string, in rc.Params) (out rc.Params, err error) {
if err != nil {
return nil, errors.Wrap(err, "failed to make request")
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
req.Header.Set("Content-Type", "application/json")
if authUser != "" || authPass != "" {
@ -182,7 +184,7 @@ func doCall(path string, in rc.Params) (out rc.Params, err error) {
}
// Run the remote control command passed in
func run(args []string) (err error) {
func run(ctx context.Context, args []string) (err error) {
path := strings.Trim(args[0], "/")
// parse input
@ -208,7 +210,7 @@ func run(args []string) (err error) {
}
// Do the call
out, callErr := doCall(path, in)
out, callErr := doCall(ctx, path, in)
// Write the JSON blob to stdout if required
if out != nil && !noOutput {
@ -222,8 +224,8 @@ func run(args []string) (err error) {
}
// List the available commands to stdout
func list() error {
list, err := doCall("rc/list", nil)
func list(ctx context.Context) error {
list, err := doCall(ctx, "rc/list", nil)
if err != nil {
return errors.Wrap(err, "failed to list")
}

3
lib/rest/rest.go
View File

@ -212,10 +212,11 @@ func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, e
if opts.ContentLength != nil && *opts.ContentLength == 0 {
body = nil
}
req, err := http.NewRequestWithContext(ctx, opts.Method, url, body)
req, err := http.NewRequest(opts.Method, url, body)
if err != nil {
return
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
headers := make(map[string]string)
// Set default headers
for k, v := range api.headers {