From 422ad38e5b618ec3158b171731d384da41db7732 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 18 Dec 2019 17:02:13 +0000 Subject: [PATCH] copyurl: add --stdout flag to write to stdout --- cmd/copyurl/copyurl.go | 44 +++++++++++++++++++++++--------- fs/operations/operations.go | 37 ++++++++++++++++++++++----- fs/operations/operations_test.go | 31 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index 21118ba6e..7c4aedd5c 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -2,6 +2,8 @@ package copyurl import ( "context" + "errors" + "os" "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs" @@ -12,37 +14,55 @@ import ( var ( autoFilename = false + stdout = false ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() - flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the url and use it for destination file path") + flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path") + flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file") } var commandDefinition = &cobra.Command{ Use: "copyurl https://example.com dest:path", Short: `Copy url content to dest.`, Long: ` -Download urls content and copy it to destination -without saving it in tmp storage. +Download a URL's content and copy it to the destination without saving +it in temporary storage. -Setting --auto-filename flag will cause retrieving file name from url and using it in destination path. +Setting --auto-filename will cause the file name to be retreived from +the from URL (after any redirections) and used in the destination +path. + +Setting --stdout or making the output file name "-" will cause the +output to be written to standard output. `, - Run: func(command *cobra.Command, args []string) { - cmd.CheckArgs(2, 2, command, args) + RunE: func(command *cobra.Command, args []string) (err error) { + cmd.CheckArgs(1, 2, command, args) var dstFileName string var fsdst fs.Fs - if autoFilename { - fsdst = cmd.NewFsDir(args[1:]) - } else { - fsdst, dstFileName = cmd.NewFsDstFile(args[1:]) + if !stdout { + if len(args) < 2 { + return errors.New("need 2 arguments if not using --stdout") + } + if args[1] == "-" { + stdout = true + } else if autoFilename { + fsdst = cmd.NewFsDir(args[1:]) + } else { + fsdst, dstFileName = cmd.NewFsDstFile(args[1:]) + } } - cmd.Run(true, true, command, func() error { - _, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename) + if stdout { + err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout) + } else { + _, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename) + } return err }) + return nil }, } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 3fd29ae3b..ff4d970a1 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "path" "path/filepath" "sort" @@ -1616,26 +1617,48 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo return obj, nil } -// CopyURL copies the data from the url to (fdst, dstFileName) -func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) { +// copyURLFunc is called from CopyURLFn +type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) + +// copyURLFn copies the data from the url to the function supplied +func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) { client := fshttp.NewClient(fs.Config) resp, err := client.Get(url) if err != nil { - return nil, err + return err } defer fs.CheckClose(resp.Body, &err) if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, errors.Errorf("CopyURL failed: %s", resp.Status) + return errors.Errorf("CopyURL failed: %s", resp.Status) + } + modTime, err := http.ParseTime(resp.Header.Get("Last-Modified")) + if err != nil { + modTime = time.Now() } - if dstFileNameFromURL { dstFileName = path.Base(resp.Request.URL.Path) if dstFileName == "." || dstFileName == "/" { - return nil, errors.Errorf("CopyURL failed: file name wasn't found in url") + return errors.Errorf("CopyURL failed: file name wasn't found in url") } } + return fn(ctx, dstFileName, resp.Body, resp.ContentLength, modTime) +} - return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now()) +// CopyURL copies the data from the url to (fdst, dstFileName) +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) { + err = copyURLFn(ctx, dstFileName, url, dstFileNameFromURL, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime) + return err + }) + return dst, err +} + +// CopyURLToWriter copies the data from the url to the io.Writer supplied +func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) { + return copyURLFn(ctx, "", url, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + _, err = io.Copy(out, in) + return err + }) } // BackupDir returns the correctly configured --backup-dir diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 8826f3767..7d76485a7 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -665,6 +665,37 @@ func TestCopyURL(t *testing.T) { fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) } +func TestCopyURLToWriter(t *testing.T) { + contents := "file contents\n" + + // check when reading from regular HTTP server + status := 0 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != 0 { + http.Error(w, "an error ocurred", status) + return + } + _, err := w.Write([]byte(contents)) + assert.NoError(t, err) + }) + ts := httptest.NewServer(handler) + defer ts.Close() + + // test normal fetch + var buf bytes.Buffer + err := operations.CopyURLToWriter(context.Background(), ts.URL, &buf) + require.NoError(t, err) + assert.Equal(t, contents, buf.String()) + + // test fetch with error + status = http.StatusNotFound + buf.Reset() + err = operations.CopyURLToWriter(context.Background(), ts.URL, &buf) + require.Error(t, err) + assert.Contains(t, err.Error(), "Not Found") + assert.Equal(t, 0, len(buf.String())) +} + func TestMoveFile(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise()