yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make NewObject return an error 

* make it return an error
  * make a canonical error fs.ErrorNotFound
  * make a test for it
  * remove logs/debugs of error
s3-about
Nick Craig-Wood 2016-06-25 21:23:20 +01:00
parent b1f131964e
commit ab43005422
13 changed files with 268 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
amazonclouddrive/amazonclouddrive.go
View File

@ -215,10 +215,13 @@ func NewFs(name, root string) (fs.Fs, error) {
// No root so return old f
return f, nil
}
obj := newF.newObjectWithInfo(remote, nil)
if obj == nil {
// File doesn't exist so return old f
return f, nil
_, err := newF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
return f, nil
}
return nil, err
}
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
@ -228,8 +231,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -240,17 +243,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *acd.Node) fs.Object {
} else {
err := o.readMetaData() // reads info and meta, returning an error
if err != nil {
fs.Log(o, "Failed to read metadata: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -384,10 +385,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
}
case fileKind:
if o := f.newObjectWithInfo(remote, node); o != nil {
if out.Add(o) {
return true
}
o, err := f.newObjectWithInfo(remote, node)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
return true
}
default:
// ignore ASSET etc
@ -430,7 +434,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
switch err {
case nil:
return o, o.Update(in, src)
case fs.ErrorDirNotFound, acd.ErrorNodeNotFound:
case fs.ErrorObjectNotFound:
// Not found so create it
default:
return nil, err
@ -608,12 +612,17 @@ func (o *Object) Size() int64 {
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (o *Object) readMetaData() (err error) {
if o.info != nil {
return nil
}
leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err
}
folder := acd.FolderFromId(directoryID, o.fs.c.Nodes)
@ -624,6 +633,9 @@ func (o *Object) readMetaData() (err error) {
return o.fs.shouldRetry(resp, err)
})
if err != nil {
if err == acd.ErrorNodeNotFound {
return fs.ErrorObjectNotFound
}
return err
}
o.info = info.Node

49
b2/b2.go
View File

@ -237,12 +237,17 @@ func NewFs(name, root string) (fs.Fs, error) {
} else {
f.root += "/"
}
obj := f.NewObject(remote)
if obj != nil {
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
_, err := f.NewObject(remote)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
f.root = oldRoot
return f, nil
}
return nil, err
}
f.root = oldRoot
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
}
return f, nil
}
@ -321,8 +326,8 @@ func (f *Fs) clearUploadURL() {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *api.File) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.File) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -330,23 +335,20 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.File) fs.Object {
if info != nil {
err := o.decodeMetaData(info)
if err != nil {
fs.Debug(o, "Failed to decode metadata: %s", err)
return nil
return nil, err
}
} else {
err := o.readMetaData() // reads info and headers, returning an error
if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -495,10 +497,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted
}
} else {
if o := f.newObjectWithInfo(remote, object); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
@ -855,10 +859,13 @@ func (o *Object) readMetaData() (err error) {
return errEndList // read only 1 item
})
if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err
}
if info == nil {
return errors.Errorf("object %q not found", o.remote)
return fs.ErrorObjectNotFound
}
return o.decodeMetaData(info)
}

64
drive/drive.go
View File

@ -80,7 +80,6 @@ var (
"text/plain": "txt",
}
extensionToMimeType map[string]string
errorObjectNotFound = errors.New("Object not found")
)
// Register with Fs
@ -337,7 +336,7 @@ func NewFs(name, path string) (fs.Fs, error) {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfoErr(remote, nil)
_, err := newF.newObjectWithInfo(remote, nil)
if err != nil {
// File doesn't exist so return old f
return f, nil
@ -350,7 +349,9 @@ func NewFs(name, path string) (fs.Fs, error) {
}
// Return an Object from a path
func (f *Fs) newObjectWithInfoErr(remote string, info *drive.File) (fs.Object, error) {
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -366,21 +367,9 @@ func (f *Fs) newObjectWithInfoErr(remote string, info *drive.File) (fs.Object, e
return o, nil
}
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) fs.Object {
o, err := f.newObjectWithInfoErr(remote, info)
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
}
return o
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -478,10 +467,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
case item.Md5Checksum != "":
// If item has MD5 sum it is a file stored on drive
if o := f.newObjectWithInfo(remote, item); o != nil {
if out.Add(o) {
return true
}
o, err := f.newObjectWithInfo(remote, item)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
return true
}
case len(item.ExportLinks) != 0:
// If item has export links then it is a google doc
@ -489,14 +481,17 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
if extension == "" {
fs.Debug(remote, "No export formats found")
} else {
if o := f.newObjectWithInfo(remote+"."+extension, item); o != nil {
obj := o.(*Object)
obj.isDocument = true
obj.url = link
obj.bytes = -1
if out.Add(o) {
return true
}
o, err := f.newObjectWithInfo(remote+"."+extension, item)
if err != nil {
out.SetError(err)
return true
}
obj := o.(*Object)
obj.isDocument = true
obj.url = link
obj.bytes = -1
if out.Add(o) {
return true
}
}
default:
@ -547,11 +542,11 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje
//
// The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
exisitingObj, err := f.newObjectWithInfoErr(src.Remote(), nil)
exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
switch err {
case nil:
return exisitingObj, exisitingObj.Update(in, src)
case fs.ErrorDirNotFound, errorObjectNotFound:
case fs.ErrorObjectNotFound:
// Not found so create it
return f.PutUnchecked(in, src)
default:
@ -857,6 +852,9 @@ func (o *Object) readMetaData() (err error) {
leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return fs.ErrorObjectNotFound
}
return err
}
@ -871,7 +869,7 @@ func (o *Object) readMetaData() (err error) {
return err
}
if !found {
return errorObjectNotFound
return fs.ErrorObjectNotFound
}
return nil
}

57
dropbox/dropbox.go
View File

@ -13,6 +13,7 @@ import (
"io"
"io/ioutil"
"log"
"net/http"
"path"
"regexp"
"strings"
@ -198,8 +199,8 @@ func (f *Fs) setRoot(root string) {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -209,17 +210,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) fs.Object {
} else {
err := o.readEntryAndSetMetadata()
if err != nil {
// logged already fs.Debug("Failed to read info: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -322,10 +321,13 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
out.SetError(err)
return
}
if o := f.newObjectWithInfo(path, entry); o != nil {
if out.Add(o) {
return
}
o, err := f.newObjectWithInfo(path, entry)
if err != nil {
out.SetError(err)
return
}
if out.Add(o) {
return
}
} else {
nameTree.PutFile(parentPath, lastComponent, entry)
@ -345,10 +347,12 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
if err != nil {
return err
}
if o := f.newObjectWithInfo(path, entry); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(path, entry)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
return nil
}
@ -387,10 +391,13 @@ func (f *Fs) listOneLevel(out fs.ListOpts, dir string) {
return
}
} else {
if o := f.newObjectWithInfo(remote, entry); o != nil {
if out.Add(o) {
return
}
o, err := f.newObjectWithInfo(remote, entry)
if err != nil {
out.SetError(err)
return
}
if out.Add(o) {
return
}
}
}
@ -622,8 +629,12 @@ func (o *Object) setMetadataFromEntry(info *dropbox.Entry) {
func (o *Object) readEntry() (*dropbox.Entry, error) {
entry, err := o.fs.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit)
if err != nil {
fs.Debug(o, "Error reading file: %s", err)
return nil, errors.Wrap(err, "error reading file")
if dropboxErr, ok := err.(*dropbox.Error); ok {
if dropboxErr.StatusCode == http.StatusNotFound {
return nil, fs.ErrorObjectNotFound
}
}
return nil, err
}
return entry, nil
}

6
fs/fs.go
View File

@ -39,6 +39,7 @@ var (
ErrorDirExists = errors.New("can't copy directory - destination already exists")
ErrorCantSetModTime = errors.New("can't set modified time")
ErrorDirNotFound = errors.New("directory not found")
ErrorObjectNotFound = errors.New("object not found")
ErrorLevelNotSupported = errors.New("level value not supported")
ErrorListAborted = errors.New("list aborted")
ErrorListOnlyRoot = errors.New("can only list from root")
@ -117,8 +118,9 @@ type Fs interface {
Info
ListFser
// NewObject finds the Object at remote. Returns nil if can't be found
NewObject(remote string) Object
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
NewObject(remote string) (Object, error)
// Put in to the remote path with the modTime given of the given size
//

26
fstest/fstests/fstests.go
View File

@ -171,23 +171,29 @@ func TestFsListDirEmpty(t *testing.T) {
// TestFsNewObjectNotFound tests not finding a object
func TestFsNewObjectNotFound(t *testing.T) {
skipIfNotOk(t)
if remote.NewObject("potato") != nil {
t.Fatal("Didn't expect to find object")
}
// Object in an existing directory
o, err := remote.NewObject("potato")
assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err)
// Now try an object in a non existing directory
o, err = remote.NewObject("directory/not/found/potato")
assert.Nil(t, o)
assert.Equal(t, fs.ErrorObjectNotFound, err)
}
func findObject(t *testing.T, Name string) fs.Object {
var obj fs.Object
var err error
for i := 1; i <= eventualConsistencyRetries; i++ {
obj = remote.NewObject(Name)
if obj != nil {
obj, err = remote.NewObject(Name)
if err == nil {
break
}
t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d", i, eventualConsistencyRetries)
t.Logf("Sleeping for 1 second for findObject eventual consistency: %d/%d (%v)", i, eventualConsistencyRetries, err)
time.Sleep(1 * time.Second)
}
if obj == nil {
t.Fatalf("Object not found: %q", Name)
if err != nil {
t.Fatalf("Object %q not found: %v", Name, err)
}
return obj
}
@ -629,10 +635,6 @@ func TestFsIsFileNotFound(t *testing.T) {
t.Fatalf("Failed to make remote %q: %v", remoteName, err)
}
fstest.CheckListing(t, fileRemote, []fstest.Item{})
_, ok := fileRemote.(*fs.Limited)
if ok {
t.Errorf("%v is is a fs.Limited", fileRemote)
}
}
// TestObjectRemove tests Remove

32
googlecloudstorage/googlecloudstorage.go
View File

@ -267,8 +267,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -278,17 +278,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) fs.Object {
} else {
err := o.readMetaData() // reads info and meta, returning an error
if err != nil {
// logged already FsDebug("Failed to read info: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -369,10 +367,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted
}
} else {
if o := f.newObjectWithInfo(remote, object); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
@ -596,7 +596,11 @@ func (o *Object) readMetaData() (err error) {
}
object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
if err != nil {
fs.Debug(o, "Failed to read info: %s", err)
if gErr, ok := err.(*googleapi.Error); ok {
if gErr.Code == http.StatusNotFound {
return fs.ErrorObjectNotFound
}
}
return err
}
o.setMetaData(object)

35
local/local.go
View File

@ -76,12 +76,7 @@ func NewFs(name, root string) (fs.Fs, error) {
fi, err := os.Lstat(f.root)
if err == nil && fi.Mode().IsRegular() {
// It is a file, so use the parent as the root
var remote string
f.root, remote = getDirFile(f.root)
obj := f.NewObject(remote)
if obj == nil {
return nil, errors.Errorf("failed to make object for %q in %q", remote, f.root)
}
f.root, _ = getDirFile(f.root)
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
}
@ -118,24 +113,25 @@ func (f *Fs) newObject(remote string) *Object {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) fs.Object {
func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) {
o := f.newObject(remote)
if info != nil {
o.info = info
} else {
err := o.lstat()
if err != nil {
fs.Debug(o, "Failed to stat %s: %s", o.path, err)
return nil
if os.IsNotExist(err) {
return nil, fs.ErrorObjectNotFound
}
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -192,10 +188,13 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
}
}
} else {
if fso := f.newObjectWithInfo(newRemote, fi); fso != nil {
if fso.Storable() && out.Add(fso) {
return nil
}
fso, err := f.newObjectWithInfo(newRemote, fi)
if err != nil {
out.SetError(err)
return nil
}
if fso.Storable() && out.Add(fso) {
return nil
}
}
}

44
onedrive/onedrive.go
View File

@ -206,10 +206,13 @@ func NewFs(name, root string) (fs.Fs, error) {
// No root so return old f
return f, nil
}
obj := newF.newObjectWithInfo(remote, nil)
if obj == nil {
// File doesn't exist so return old f
return f, nil
_, err := newF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
return f, nil
}
return nil, err
}
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
@ -227,8 +230,8 @@ func (f *Fs) rootSlash() string {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *api.Item) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.Item) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -239,17 +242,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Item) fs.Object {
} else {
err := o.readMetaData() // reads info and meta, returning an error
if err != nil {
// logged already FsDebug("Failed to read info: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -391,10 +392,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
}
} else {
if o := f.newObjectWithInfo(remote, info); o != nil {
if out.Add(o) {
return true
}
o, err := f.newObjectWithInfo(remote, info)
if err != nil {
out.SetError(err)
return true
}
if out.Add(o) {
return true
}
}
return false
@ -705,7 +709,11 @@ func (o *Object) readMetaData() (err error) {
// }
info, _, err := o.fs.readMetaDataForPath(o.srvPath())
if err != nil {
fs.Debug(o, "Failed to read info: %s", err)
if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "itemNotFound" {
return fs.ErrorObjectNotFound
}
}
return err
}
o.setMetaData(info)

5
rclone.go
View File

@ -383,8 +383,9 @@ func NewFsSrc(remote string) fs.Fs {
log.Fatalf("Can't limit to single files when using filters: %v", remote)
}
// Limit transfers to this file
fs.Config.Filter.AddFile(path.Base(fsPath))
} else if err != nil {
err = fs.Config.Filter.AddFile(path.Base(fsPath))
}
if err != nil {
fs.Stats.Error()
log.Fatalf("Failed to create file system for %q: %v", remote, err)
}

34
s3/s3.go
View File

@ -342,8 +342,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) fs.Object {
//If it can't be found it returns the error ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -361,17 +361,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *s3.Object) fs.Object {
} else {
err := o.readMetaData() // reads info and meta, returning an error
if err != nil {
// logged already FsDebug("Failed to read info: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -482,10 +480,12 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted
}
} else {
if o := f.newObjectWithInfo(remote, object); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
@ -634,7 +634,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
if err != nil {
return nil, err
}
return f.NewObject(remote), err
return f.NewObject(remote)
}
// Hashes returns the supported hash sets.
@ -697,7 +697,11 @@ func (o *Object) readMetaData() (err error) {
}
resp, err := o.fs.c.HeadObject(&req)
if err != nil {
fs.Debug(o, "Failed to read info: %s", err)
if awsErr, ok := err.(awserr.RequestFailure); ok {
if awsErr.StatusCode() == http.StatusNotFound {
return fs.ErrorObjectNotFound
}
}
return err
}
var size int64

45
swift/swift.go
View File

@ -223,8 +223,8 @@ func NewFs(name, root string) (fs.Fs, error) {
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -241,17 +241,15 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) fs.Object {
} else {
err := o.readMetaData() // reads info and headers, returning an error
if err != nil {
fs.Debug(o, "Failed to read metadata: %s", err)
return nil
return nil, err
}
}
return o
return o, nil
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found it
// returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
@ -331,12 +329,14 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) {
return fs.ErrorListAborted
}
} else {
if o := f.newObjectWithInfo(remote, object); o != nil {
// Storable does a full metadata read on 0 size objects which might be dynamic large objects
if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
// Storable does a full metadata read on 0 size objects which might be dynamic large objects
if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
}
}
}
@ -437,9 +437,11 @@ func (f *Fs) Purge() error {
go func() {
err = f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
if !isDirectory {
if o := f.newObjectWithInfo(remote, object); o != nil {
toBeDeleted <- o
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
toBeDeleted <- o
}
return nil
})
@ -472,7 +474,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
if err != nil {
return nil, err
}
return f.NewObject(remote), nil
return f.NewObject(remote)
}
// Hashes returns the supported hash sets.
@ -525,7 +527,7 @@ func (o *Object) Hash(t fs.HashType) (string, error) {
func (o *Object) hasHeader(header string) (bool, error) {
err := o.readMetaData()
if err != nil {
if err == swift.ObjectNotFound {
if err == fs.ErrorObjectNotFound {
return false, nil
}
return false, err
@ -552,12 +554,17 @@ func (o *Object) Size() int64 {
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
//
// it returns fs.ErrorObjectNotFound if the object isn't found
func (o *Object) readMetaData() (err error) {
if o.headers != nil {
return nil
}
info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
if err != nil {
if err == swift.ObjectNotFound {
return fs.ErrorObjectNotFound
}
return err
}
o.info = info

31
yandex/yandex.go
View File

@ -263,10 +263,12 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
return fs.ErrorListAborted
}
} else {
if o := f.newObjectWithInfo(remote, object); o != nil {
if out.Add(o) {
return fs.ErrorListAborted
}
o, err := f.newObjectWithInfo(remote, object)
if err != nil {
return err
}
if out.Add(o) {
return fs.ErrorListAborted
}
}
return nil
@ -295,17 +297,16 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
}
}
// NewObject returns an Object from a path
//
// May return nil if an error occurred
func (f *Fs) NewObject(remote string) fs.Object {
// NewObject finds the Object at remote. If it can't be found it
// returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
// Return an Object from a path
//
// May return nil if an error occurred
func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) fs.Object {
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@ -315,11 +316,10 @@ func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse)
} else {
err := o.readMetaData()
if err != nil {
fs.Debug(f, "Couldn't get object '%s' metadata: %s", o.remotePath(), err)
return nil
return nil, err
}
}
return o
return o, nil
}
// setMetaData sets the fs data from a storage.Object
@ -355,6 +355,11 @@ func (o *Object) readMetaData() (err error) {
var opt2 yandex.ResourceInfoRequestOptions
ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec()
if err != nil {
if dcErr, ok := err.(yandex.DiskClientError); ok {
if dcErr.Code == "DiskNotFoundError" {
return fs.ErrorObjectNotFound
}
}
return err
}
o.setMetaData(ResourceInfoResponse)