yves
/
rclone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

vfs: factor the vfs cache into its own package

vfs-used-is-size
Nick Craig-Wood 2020-02-28 14:44:15 +00:00
parent fd39cbc193
commit eed9c5738d
21 changed files with 423 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/cmount/fs.go
View File

@ -361,12 +361,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
if errc != 0 {
return errc
}
var err error
if fsys.VFS.Opt.CacheMode < vfs.CacheModeWrites || handle.Node().Mode()&os.ModeAppend == 0 {
n, err = handle.WriteAt(buff, ofst)
} else {
n, err = handle.Write(buff)
}
n, err := handle.WriteAt(buff, ofst)
if err != nil {
return translateError(err)
}

8
cmd/mount/handle.go
View File

@ -5,7 +5,6 @@ package mount
import (
"context"
"io"
"os"
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
@ -42,12 +41,7 @@ var _ fusefs.HandleWriter = (*FileHandle)(nil)
// Write data to the file handle
func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) (err error) {
defer log.Trace(fh, "len=%d, offset=%d", len(req.Data), req.Offset)("written=%d, err=%v", &resp.Size, &err)
var n int
if fh.Handle.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || fh.Handle.Node().Mode()&os.ModeAppend == 0 {
n, err = fh.Handle.WriteAt(req.Data, req.Offset)
} else {
n, err = fh.Handle.Write(req.Data)
}
n, err := fh.Handle.WriteAt(req.Data, req.Offset)
if err != nil {
return translateError(err)
}

7
cmd/mount2/file.go
View File

@ -6,7 +6,6 @@ import (
"context"
"fmt"
"io"
"os"
"syscall"
fusefs "github.com/hanwen/go-fuse/v2/fs"
@ -74,11 +73,7 @@ func (f *FileHandle) Write(ctx context.Context, data []byte, off int64) (written
var n int
var err error
defer log.Trace(f, "off=%d", off)("n=%d, off=%d, errno=%v", &n, &off, &errno)
if f.h.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || f.h.Node().Mode()&os.ModeAppend == 0 {
n, err = f.h.WriteAt(data, off)
} else {
n, err = f.h.Write(data)
}
n, err = f.h.WriteAt(data, off)
return uint32(n), translateError(err)
}

5
vfs/dir.go
View File

@ -16,6 +16,7 @@ import (
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/vfs/vfscommon"
)
// Dir represents a directory entry
@ -150,7 +151,7 @@ func (d *Dir) changeNotify(relativePath string, entryType fs.EntryType) {
d.mu.RLock()
absPath := path.Join(d.path, relativePath)
d.mu.RUnlock()
d.invalidateDir(findParent(absPath))
d.invalidateDir(vfscommon.FindParent(absPath))
if entryType == fs.EntryDirectory {
d.invalidateDir(absPath)
}
@ -168,7 +169,7 @@ func (d *Dir) ForgetPath(relativePath string, entryType fs.EntryType) {
absPath := path.Join(d.path, relativePath)
d.mu.RUnlock()
if absPath != "" {
d.invalidateDir(findParent(absPath))
d.invalidateDir(vfscommon.FindParent(absPath))
}
if entryType == fs.EntryDirectory {
d.forgetDirPath(relativePath)

23
vfs/file.go
View File

@ -12,6 +12,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/vfs/vfscommon"
)
// The File object is tightly coupled to the Dir object. Since they
@ -124,7 +125,7 @@ func (f *File) Path() string {
// osPath returns the full path of the file in the cache in OS format
func (f *File) osPath() string {
return f.d.vfs.cache.toOSPath(f.Path())
return f.d.vfs.cache.ToOSPath(f.Path())
}
// Sys returns underlying data source (can be nil) - satisfies Node interface
@ -220,8 +221,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
}
// Rename in the cache if it exists
if f.d.vfs.Opt.CacheMode != CacheModeOff && f.d.vfs.cache.exists(f.Path()) {
if err := f.d.vfs.cache.rename(f.Path(), newPath); err != nil {
if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f.Path()) {
if err := f.d.vfs.cache.Rename(f.Path(), newPath); err != nil {
fs.Infof(f.Path(), "File.Rename failed in Cache: %v", err)
}
}
@ -403,8 +404,8 @@ func (f *File) _applyPendingModTime() error {
}
// set the time of the file in the cache
if f.d.vfs.Opt.CacheMode != CacheModeOff {
f.d.vfs.cache.setModTime(f._path(), f.pendingModTime)
if f.d.vfs.cache != nil {
f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
}
// set the time of the object
@ -580,8 +581,8 @@ func (f *File) Remove() error {
// called with File.mu released
d.delObject(f.Name())
// Remove the object from the cache
if d.vfs.Opt.CacheMode >= CacheModeMinimal {
d.vfs.cache.remove(f.Path())
if d.vfs.cache != nil {
d.vfs.cache.Remove(f.Path())
}
return nil
}
@ -679,10 +680,10 @@ func (f *File) Open(flags int) (fd Handle, err error) {
d := f.d
f.mu.RUnlock()
CacheMode := d.vfs.Opt.CacheMode
if CacheMode >= CacheModeMinimal && (d.vfs.cache.opens(f.Path()) > 0 || d.vfs.cache.exists(f.Path())) {
if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.Opens(f.Path()) > 0 || d.vfs.cache.Exists(f.Path())) {
fd, err = f.openRW(flags)
} else if read && write {
if CacheMode >= CacheModeMinimal {
if CacheMode >= vfscommon.CacheModeMinimal {
fd, err = f.openRW(flags)
} else {
// Open write only and hope the user doesn't
@ -691,13 +692,13 @@ func (f *File) Open(flags int) (fd Handle, err error) {
fd, err = f.openWrite(flags)
}
} else if write {
if CacheMode >= CacheModeWrites {
if CacheMode >= vfscommon.CacheModeWrites {
fd, err = f.openRW(flags)
} else {
fd, err = f.openWrite(flags)
}
} else if read {
if CacheMode >= CacheModeFull {
if CacheMode >= vfscommon.CacheModeFull {
fd, err = f.openRW(flags)
} else {
fd, err = f.openRead()

41
vfs/file_test.go
View File

@ -11,12 +11,13 @@ import (
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/mockfs"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstest.Item) {
opt := DefaultOpt
func fileCreate(t *testing.T, r *fstest.Run, mode vfscommon.CacheMode) (*VFS, *File, fstest.Item) {
opt := vfscommon.DefaultOpt
opt.CacheMode = mode
vfs := New(r.Fremote, &opt)
@ -33,7 +34,7 @@ func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstes
func TestFileMethods(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
vfs, file, _ := fileCreate(t, r, CacheModeOff)
vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
// String
assert.Equal(t, "dir/file1", file.String())
@ -88,7 +89,7 @@ func TestFileSetModTime(t *testing.T) {
return
}
defer r.Finalise()
vfs, file, file1 := fileCreate(t, r, CacheModeOff)
vfs, file, file1 := fileCreate(t, r, vfscommon.CacheModeOff)
err := file.SetModTime(t2)
require.NoError(t, err)
@ -115,7 +116,7 @@ func fileCheckContents(t *testing.T, file *File) {
func TestFileOpenRead(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
_, file, _ := fileCreate(t, r, CacheModeOff)
_, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
fileCheckContents(t, file)
}
@ -168,7 +169,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {
func TestFileOpenWrite(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
vfs, file, _ := fileCreate(t, r, CacheModeOff)
vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
fd, err := file.openWrite(os.O_WRONLY | os.O_TRUNC)
require.NoError(t, err)
@ -189,7 +190,7 @@ func TestFileOpenWrite(t *testing.T) {
func TestFileRemove(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
vfs, file, _ := fileCreate(t, r, CacheModeOff)
vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
err := file.Remove()
require.NoError(t, err)
@ -204,7 +205,7 @@ func TestFileRemove(t *testing.T) {
func TestFileRemoveAll(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
vfs, file, _ := fileCreate(t, r, CacheModeOff)
vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
err := file.RemoveAll()
require.NoError(t, err)
@ -219,7 +220,7 @@ func TestFileRemoveAll(t *testing.T) {
func TestFileOpen(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
_, file, _ := fileCreate(t, r, CacheModeOff)
_, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
fd, err := file.Open(os.O_RDONLY)
require.NoError(t, err)
@ -242,7 +243,7 @@ func TestFileOpen(t *testing.T) {
assert.Equal(t, EPERM, err)
}
func testFileRename(t *testing.T, mode CacheMode) {
func testFileRename(t *testing.T, mode vfscommon.CacheMode) {
r := fstest.NewRun(t)
defer r.Finalise()
vfs, file, item := fileCreate(t, r, mode)
@ -255,10 +256,10 @@ func testFileRename(t *testing.T, mode CacheMode) {
require.NoError(t, err)
// check file in cache
if mode != CacheModeOff {
if mode != vfscommon.CacheModeOff {
// read contents to get file in cache
fileCheckContents(t, file)
assert.True(t, vfs.cache.exists(item.Path))
assert.True(t, vfs.cache.Exists(item.Path))
}
dir := file.Dir()
@ -274,8 +275,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
fstest.CheckItems(t, r.Fremote, item)
// check file in cache
if mode != CacheModeOff {
assert.True(t, vfs.cache.exists(item.Path))
if mode != vfscommon.CacheModeOff {
assert.True(t, vfs.cache.Exists(item.Path))
}
// check file exists in the vfs layer at its new name
@ -290,8 +291,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
fstest.CheckItems(t, r.Fremote, item)
// check file in cache
if mode != CacheModeOff {
assert.True(t, vfs.cache.exists(item.Path))
if mode != vfscommon.CacheModeOff {
assert.True(t, vfs.cache.Exists(item.Path))
}
// now try renaming it with the file open
@ -308,8 +309,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
newItem := fstest.NewItem("newLeaf", string(newContents), item.ModTime)
// check file has been renamed immediately in the cache
if mode != CacheModeOff {
assert.True(t, vfs.cache.exists("newLeaf"))
if mode != vfscommon.CacheModeOff {
assert.True(t, vfs.cache.Exists("newLeaf"))
}
// check file exists in the vfs layer at its new name
@ -326,9 +327,9 @@ func testFileRename(t *testing.T, mode CacheMode) {
func TestFileRename(t *testing.T) {
t.Run("CacheModeOff", func(t *testing.T) {
testFileRename(t, CacheModeOff)
testFileRename(t, vfscommon.CacheModeOff)
})
t.Run("CacheModeFull", func(t *testing.T) {
testFileRename(t, CacheModeFull)
testFileRename(t, vfscommon.CacheModeFull)
})
}

50
vfs/read_write.go
View File

@ -12,7 +12,6 @@ import (
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/file"
)
@ -55,12 +54,12 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
}
// mark the file as open in the cache - must be done before the mkdir
fh.d.VFS().cache.open(fh.file.Path())
fh.d.VFS().cache.Open(fh.file.Path())
// Make a place for the file
_, err = d.VFS().cache.mkdir(fh.file.Path())
_, err = d.VFS().cache.Mkdir(fh.file.Path())
if err != nil {
fh.d.VFS().cache.close(fh.file.Path())
fh.d.VFS().cache.Close(fh.file.Path())
return nil, errors.Wrap(err, "open RW handle failed to make cache directory")
}
@ -80,16 +79,6 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
return fh, nil
}
// copy an object to or from the remote while accounting for it
func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
if operations.NeedTransfer(context.TODO(), dst, src) {
newDst, err = operations.Copy(context.TODO(), f, dst, remote, src)
} else {
newDst = dst
}
return newDst, err
}
// openPending opens the file if there is a pending open
//
// call with the lock held
@ -110,12 +99,9 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
// If the remote object exists AND its cached file exists locally AND there are no
// other RW handles with it open, then attempt to update it.
if o != nil && fh.file.rwOpens() == 0 {
cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path())
if err == nil && cacheObj != nil {
_, err = copyObj(fh.d.VFS().cache.f, cacheObj, fh.file.Path(), o)
if err != nil {
return errors.Wrap(err, "open RW handle failed to update cached file")
}
err = fh.d.VFS().cache.Check(context.TODO(), o, fh.file.Path())
if err != nil {
return errors.Wrap(err, "open RW handle failed to check cache file")
}
}
@ -125,7 +111,7 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
// cache file does not exist, so need to fetch it if we have an object to fetch
// it from
if o != nil {
_, err = copyObj(fh.d.VFS().cache.f, nil, fh.file.Path(), o)
err = fh.d.VFS().cache.Fetch(context.TODO(), o, fh.file.Path())
if err != nil {
cause := errors.Cause(err)
if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound {
@ -276,22 +262,8 @@ func (fh *RWFileHandle) flushWrites(closeFile bool) error {
}
if isCopied {
// Transfer the temp file to the remote
cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path())
o, err := fh.d.VFS().cache.Store(context.TODO(), fh.file.getObject(), fh.file.Path())
if err != nil {
err = errors.Wrap(err, "failed to find cache file")
fs.Errorf(fh.logPrefix(), "%v", err)
return err
}
objPath := fh.file.Path()
objOld := fh.file.getObject()
if objOld != nil {
objPath = objOld.Remote() // use the path of the actual object if available
}
o, err := copyObj(fh.d.VFS().f, objOld, objPath, cacheObj)
if err != nil {
err = errors.Wrap(err, "failed to transfer file from cache to remote")
fs.Errorf(fh.logPrefix(), "%v", err)
return err
}
@ -322,7 +294,7 @@ func (fh *RWFileHandle) close() (err error) {
if fh.opened {
fh.file.delRWOpen()
}
fh.d.VFS().cache.close(fh.file.Path())
fh.d.VFS().cache.Close(fh.file.Path())
}()
return fh.flushWrites(true)
@ -501,6 +473,10 @@ func (fh *RWFileHandle) Write(b []byte) (n int, err error) {
// WriteAt bytes to the file at off
func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) {
if fh.flags&os.O_APPEND != 0 {
// if append is set, call Write as WriteAt returns an error if append is set
return fh.Write(b)
}
err = fh.writeFn(func() error {
n, err = fh.File.WriteAt(b, off)
return err

19
vfs/read_write_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -25,8 +26,8 @@ func cleanup(t *testing.T, r *fstest.Run, vfs *VFS) {
// Create a file and open it with the flags passed in
func rwHandleCreateFlags(t *testing.T, r *fstest.Run, create bool, filename string, flags int) (*VFS, *RWFileHandle) {
opt := DefaultOpt
opt.CacheMode = CacheModeFull
opt := vfscommon.DefaultOpt
opt.CacheMode = vfscommon.CacheModeFull
vfs := New(r.Fremote, &opt)
if create {
@ -661,8 +662,8 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
func TestRWFileHandleOpenTests(t *testing.T) {
r := fstest.NewRun(t)
opt := DefaultOpt
opt.CacheMode = CacheModeFull
opt := vfscommon.DefaultOpt
opt.CacheMode = vfscommon.CacheModeFull
vfs := New(r.Fremote, &opt)
defer cleanup(t, r, vfs)
@ -716,8 +717,8 @@ func TestRWCacheRename(t *testing.T) {
t.Skip("skip as can't rename files")
}
opt := DefaultOpt
opt.CacheMode = CacheModeFull
opt := vfscommon.DefaultOpt
opt.CacheMode = vfscommon.CacheModeFull
vfs := New(r.Fremote, &opt)
h, err := vfs.OpenFile("rename_me", os.O_WRONLY|os.O_CREATE, 0777)
@ -732,11 +733,11 @@ func TestRWCacheRename(t *testing.T) {
err = fh.Close()
require.NoError(t, err)
assert.True(t, vfs.cache.exists("rename_me"))
assert.True(t, vfs.cache.Exists("rename_me"))
err = vfs.Rename("rename_me", "i_was_renamed")
require.NoError(t, err)
assert.False(t, vfs.cache.exists("rename_me"))
assert.True(t, vfs.cache.exists("i_was_renamed"))
assert.False(t, vfs.cache.Exists("rename_me"))
assert.True(t, vfs.cache.Exists("i_was_renamed"))
}

71
vfs/vfs.go
View File

@ -27,7 +27,6 @@ import (
"io/ioutil"
"os"
"path"
"runtime"
"sort"
"strings"
"sync"
@ -36,32 +35,10 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/vfs/vfscache"
"github.com/rclone/rclone/vfs/vfscommon"
)
// DefaultOpt is the default values uses for Opt
var DefaultOpt = Options{
NoModTime: false,
NoChecksum: false,
NoSeek: false,
DirCacheTime: 5 * 60 * time.Second,
PollInterval: time.Minute,
ReadOnly: false,
Umask: 0,
UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
GID: ^uint32(0), // overriden for non windows in mount_unix.go
DirPerms: os.FileMode(0777),
FilePerms: os.FileMode(0666),
CacheMode: CacheModeOff,
CacheMaxAge: 3600 * time.Second,
CachePollInterval: 60 * time.Second,
ChunkSize: 128 * fs.MebiByte,
ChunkSizeLimit: -1,
CacheMaxSize: -1,
CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
WriteWait: 1000 * time.Millisecond,
ReadWait: 5 * time.Millisecond,
}
// Node represents either a directory (*Dir) or a file (*File)
type Node interface {
os.FileInfo
@ -180,8 +157,8 @@ var (
type VFS struct {
f fs.Fs
root *Dir
Opt Options
cache *cache
Opt vfscommon.Options
cache *vfscache.Cache
cancel context.CancelFunc
usageMu sync.Mutex
usageTime time.Time
@ -189,33 +166,9 @@ type VFS struct {
pollChan chan time.Duration
}
// Options is options for creating the vfs
type Options struct {
NoSeek bool // don't allow seeking if set
NoChecksum bool // don't check checksums if set
ReadOnly bool // if set VFS is read only
NoModTime bool // don't read mod times for files
DirCacheTime time.Duration // how long to consider directory listing cache valid
PollInterval time.Duration
Umask int
UID uint32
GID uint32
DirPerms os.FileMode
FilePerms os.FileMode
ChunkSize fs.SizeSuffix // if > 0 read files in chunks
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
CacheMode CacheMode
CacheMaxAge time.Duration
CacheMaxSize fs.SizeSuffix
CachePollInterval time.Duration
CaseInsensitive bool
WriteWait time.Duration // time to wait for in-sequence write
ReadWait time.Duration // time to wait for in-sequence read
}
// New creates a new VFS and root directory. If opt is nil, then
// DefaultOpt will be used
func New(f fs.Fs, opt *Options) *VFS {
func New(f fs.Fs, opt *vfscommon.Options) *VFS {
fsDir := fs.NewDir("", time.Now())
vfs := &VFS{
f: f,
@ -225,7 +178,7 @@ func New(f fs.Fs, opt *Options) *VFS {
if opt != nil {
vfs.Opt = *opt
} else {
vfs.Opt = DefaultOpt
vfs.Opt = vfscommon.DefaultOpt
}
// Mask the permissions with the umask
@ -260,15 +213,15 @@ func (vfs *VFS) Fs() fs.Fs {
}
// SetCacheMode change the cache mode
func (vfs *VFS) SetCacheMode(cacheMode CacheMode) {
func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) {
vfs.Shutdown()
vfs.cache = nil
if cacheMode > CacheModeOff {
if cacheMode > vfscommon.CacheModeOff {
ctx, cancel := context.WithCancel(context.Background())
cache, err := newCache(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
if err != nil {
fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err)
vfs.Opt.CacheMode = CacheModeOff
vfs.Opt.CacheMode = vfscommon.CacheModeOff
cancel()
return
}
@ -288,10 +241,10 @@ func (vfs *VFS) Shutdown() {
// CleanUp deletes the contents of the on disk cache
func (vfs *VFS) CleanUp() error {
if vfs.Opt.CacheMode == CacheModeOff {
if vfs.Opt.CacheMode == vfscommon.CacheModeOff {
return nil
}
return vfs.cache.cleanUp()
return vfs.cache.CleanUp()
}
// FlushDirCache empties the directory cache

5
vfs/vfs_case_test.go
View File

@ -6,6 +6,7 @@ import (
"testing"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -32,11 +33,11 @@ func TestCaseSensitivity(t *testing.T) {
file3 := r.WriteObject(ctx, "FilEb", "data3", t3)
// Create a case-Sensitive and case-INsensitive VFS
optCS := DefaultOpt
optCS := vfscommon.DefaultOpt
optCS.CaseInsensitive = false
vfsCS := New(r.Fremote, &optCS)
optCI := DefaultOpt
optCI := vfscommon.DefaultOpt
optCI.CaseInsensitive = true
vfsCI := New(r.Fremote, &optCI)

5
vfs/vfs_test.go
View File

@ -13,6 +13,7 @@ import (
_ "github.com/rclone/rclone/backend/all" // import all the backends
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -101,13 +102,13 @@ func TestVFSNew(t *testing.T) {
// Check making a VFS with nil options
vfs := New(r.Fremote, nil)
var defaultOpt = DefaultOpt
var defaultOpt = vfscommon.DefaultOpt
defaultOpt.DirPerms |= os.ModeDir
assert.Equal(t, vfs.Opt, defaultOpt)
assert.Equal(t, vfs.f, r.Fremote)
// Check the initialisation
var opt = DefaultOpt
var opt = vfscommon.DefaultOpt
opt.DirPerms = 0777
opt.FilePerms = 0666
opt.Umask = 0002

248
vfs/cache.go → vfs/vfscache/vfscache.go
View File

@ -1,10 +1,8 @@
// This deals with caching of files locally
package vfs
// Package vfscache deals with caching of files locally for the VFS layer
package vfscache
import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
@ -18,58 +16,19 @@ import (
"github.com/rclone/rclone/fs"
fscache "github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/vfs/vfscommon"
)
// CacheMode controls the functionality of the cache
type CacheMode byte
// CacheMode options
const (
CacheModeOff CacheMode = iota // cache nothing - return errors for writes which can't be satisfied
CacheModeMinimal // cache only the minimum, eg read/write opens
CacheModeWrites // cache all files opened with write intent
CacheModeFull // cache all files opened in any mode
)
var cacheModeToString = []string{
CacheModeOff: "off",
CacheModeMinimal: "minimal",
CacheModeWrites: "writes",
CacheModeFull: "full",
}
// String turns a CacheMode into a string
func (l CacheMode) String() string {
if l >= CacheMode(len(cacheModeToString)) {
return fmt.Sprintf("CacheMode(%d)", l)
}
return cacheModeToString[l]
}
// Set a CacheMode
func (l *CacheMode) Set(s string) error {
for n, name := range cacheModeToString {
if s != "" && name == s {
*l = CacheMode(n)
return nil
}
}
return errors.Errorf("Unknown cache mode level %q", s)
}
// Type of the value
func (l *CacheMode) Type() string {
return "CacheMode"
}
// cache opened files
type cache struct {
f fs.Fs // fs for the cache directory
opt *Options // vfs Options
root string // root of the cache directory
itemMu sync.Mutex // protects the following variables
item map[string]*cacheItem // files/directories in the cache
used int64 // total size of files in the cache
// Cache opened files
type Cache struct {
fremote fs.Fs // fs for the remote we are caching
fcache fs.Fs // fs for the cache directory
opt *vfscommon.Options // vfs Options
root string // root of the cache directory
itemMu sync.Mutex // protects the following variables
item map[string]*cacheItem // files/directories in the cache
used int64 // total size of files in the cache
}
// cacheItem is stored in the item map
@ -85,31 +44,32 @@ func newCacheItem(isFile bool) *cacheItem {
return &cacheItem{atime: time.Now(), isFile: isFile}
}
// newCache creates a new cache heirachy for f
// New creates a new cache heirachy for fremote
//
// This starts background goroutines which can be cancelled with the
// context passed in.
func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
fRoot := filepath.FromSlash(f.Root())
func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options) (*Cache, error) {
fRoot := filepath.FromSlash(fremote.Root())
if runtime.GOOS == "windows" {
if strings.HasPrefix(fRoot, `\\?`) {
fRoot = fRoot[3:]
}
fRoot = strings.Replace(fRoot, ":", "", -1)
}
root := filepath.Join(config.CacheDir, "vfs", f.Name(), fRoot)
root := filepath.Join(config.CacheDir, "vfs", fremote.Name(), fRoot)
fs.Debugf(nil, "vfs cache root is %q", root)
f, err := fscache.Get(root)
fcache, err := fscache.Get(root)
if err != nil {
return nil, errors.Wrap(err, "failed to create cache remote")
}
c := &cache{
f: f,
opt: opt,
root: root,
item: make(map[string]*cacheItem),
c := &Cache{
fremote: fremote,
fcache: fcache,
opt: opt,
root: root,
item: make(map[string]*cacheItem),
}
go c.cleaner(ctx)
@ -117,15 +77,6 @@ func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
return c, nil
}
// findParent returns the parent directory of name, or "" for the root
func findParent(name string) string {
parent := filepath.Dir(name)
if parent == "." || parent == "/" {
parent = ""
}
return parent
}
// clean returns the cleaned version of name for use in the index map
func clean(name string) string {
name = strings.Trim(name, "/")
@ -136,17 +87,17 @@ func clean(name string) string {
return name
}
// toOSPath turns a remote relative name into an OS path in the cache
func (c *cache) toOSPath(name string) string {
// ToOSPath turns a remote relative name into an OS path in the cache
func (c *Cache) ToOSPath(name string) string {
return filepath.Join(c.root, filepath.FromSlash(name))
}
// mkdir makes the directory for name in the cache and returns an os
// Mkdir makes the directory for name in the cache and returns an os
// path for the file
func (c *cache) mkdir(name string) (string, error) {
parent := findParent(name)
func (c *Cache) Mkdir(name string) (string, error) {
parent := vfscommon.FindParent(name)
leaf := filepath.Base(name)
parentPath := c.toOSPath(parent)
parentPath := c.ToOSPath(parent)
err := os.MkdirAll(parentPath, 0700)
if err != nil {
return "", errors.Wrap(err, "make cache directory failed")
@ -163,7 +114,7 @@ func (c *cache) mkdir(name string) (string, error) {
// name should be a remote path not an osPath
//
// must be called with itemMu held
func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
func (c *Cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
item = c.item[name]
found = item != nil
if !found {
@ -173,10 +124,10 @@ func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
return item, found
}
// opens returns the number of opens that are on the file
// Opens returns the number of opens that are on the file
//
// name should be a remote path not an osPath
func (c *cache) opens(name string) int {
func (c *Cache) Opens(name string) int {
name = clean(name)
c.itemMu.Lock()
defer c.itemMu.Unlock()
@ -190,7 +141,7 @@ func (c *cache) opens(name string) int {
// get gets name from the cache or creates a new one
//
// name should be a remote path not an osPath
func (c *cache) get(name string) *cacheItem {
func (c *Cache) get(name string) *cacheItem {
name = clean(name)
c.itemMu.Lock()
item, _ := c._get(true, name)
@ -204,7 +155,7 @@ func (c *cache) get(name string) *cacheItem {
// it also sets the size
//
// name should be a remote path not an osPath
func (c *cache) updateStat(name string, when time.Time, size int64) {
func (c *Cache) updateStat(name string, when time.Time, size int64) {
name = clean(name)
c.itemMu.Lock()
item, found := c._get(true, name)
@ -219,7 +170,7 @@ func (c *cache) updateStat(name string, when time.Time, size int64) {
// _open marks name as open, must be called with the lock held
//
// name should be a remote path not an osPath
func (c *cache) _open(isFile bool, name string) {
func (c *Cache) _open(isFile bool, name string) {
for {
item, _ := c._get(isFile, name)
item.opens++
@ -228,14 +179,14 @@ func (c *cache) _open(isFile bool, name string) {
break
}
isFile = false
name = findParent(name)
name = vfscommon.FindParent(name)
}
}
// open marks name as open
// Open marks name as open
//
// name should be a remote path not an osPath
func (c *cache) open(name string) {
func (c *Cache) Open(name string) {
name = clean(name)
c.itemMu.Lock()
c._open(true, name)
@ -245,7 +196,7 @@ func (c *cache) open(name string) {
// cacheDir marks a directory and its parents as being in the cache
//
// name should be a remote path not an osPath
func (c *cache) cacheDir(name string) {
func (c *Cache) cacheDir(name string) {
name = clean(name)
c.itemMu.Lock()
defer c.itemMu.Unlock()
@ -258,13 +209,13 @@ func (c *cache) cacheDir(name string) {
if name == "" {
break
}
name = findParent(name)
name = vfscommon.FindParent(name)
}
}
// exists checks to see if the file exists in the cache or not
func (c *cache) exists(name string) bool {
osPath := c.toOSPath(name)
// Exists checks to see if the file exists in the cache or not
func (c *Cache) Exists(name string) bool {
osPath := c.ToOSPath(name)
fi, err := os.Stat(osPath)
if err != nil {
return false
@ -276,10 +227,10 @@ func (c *cache) exists(name string) bool {
return true
}
// renames the file in cache
func (c *cache) rename(name string, newName string) (err error) {
osOldPath := c.toOSPath(name)
osNewPath := c.toOSPath(newName)
// Rename the file in cache
func (c *Cache) Rename(name string, newName string) (err error) {
osOldPath := c.ToOSPath(name)
osNewPath := c.ToOSPath(newName)
sfi, err := os.Stat(osOldPath)
if err != nil {
return errors.Wrapf(err, "Failed to stat source: %s", osOldPath)
@ -293,7 +244,7 @@ func (c *cache) rename(name string, newName string) (err error) {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "Failed to stat destination: %s", osNewPath)
}
parent := findParent(osNewPath)
parent := vfscommon.FindParent(osNewPath)
err = os.MkdirAll(parent, 0700)
if err != nil {
return errors.Wrapf(err, "Failed to create parent dir: %s", parent)
@ -321,7 +272,7 @@ func (c *cache) rename(name string, newName string) (err error) {
}
// _close marks name as closed - must be called with the lock held
func (c *cache) _close(isFile bool, name string) {
func (c *Cache) _close(isFile bool, name string) {
for {
item, _ := c._get(isFile, name)
item.opens--
@ -329,7 +280,7 @@ func (c *cache) _close(isFile bool, name string) {
if item.opens < 0 {
fs.Errorf(name, "cache: double close")
}
osPath := c.toOSPath(name)
osPath := c.ToOSPath(name)
fi, err := os.Stat(osPath)
// Update the size on close
if err == nil && !fi.IsDir() {
@ -339,23 +290,23 @@ func (c *cache) _close(isFile bool, name string) {
break
}
isFile = false
name = findParent(name)
name = vfscommon.FindParent(name)
}
}
// close marks name as closed
// Close marks name as closed
//
// name should be a remote path not an osPath
func (c *cache) close(name string) {
func (c *Cache) Close(name string) {
name = clean(name)
c.itemMu.Lock()
c._close(true, name)
c.itemMu.Unlock()
}
// remove should be called if name is deleted
func (c *cache) remove(name string) {
osPath := c.toOSPath(name)
// Remove should be called if name is deleted
func (c *Cache) Remove(name string) {
osPath := c.ToOSPath(name)
err := os.Remove(osPath)
if err != nil && !os.IsNotExist(err) {
fs.Errorf(name, "Failed to remove from cache: %v", err)
@ -366,8 +317,8 @@ func (c *cache) remove(name string) {
// removeDir should be called if dir is deleted and returns true if
// the directory is gone.
func (c *cache) removeDir(dir string) bool {
osPath := c.toOSPath(dir)
func (c *Cache) removeDir(dir string) bool {
osPath := c.ToOSPath(dir)
err := os.Remove(osPath)
if err == nil || os.IsNotExist(err) {
if err == nil {
@ -381,22 +332,22 @@ func (c *cache) removeDir(dir string) bool {
return false
}
// setModTime should be called to set the modification time of the cache file
func (c *cache) setModTime(name string, modTime time.Time) {
osPath := c.toOSPath(name)
// SetModTime should be called to set the modification time of the cache file
func (c *Cache) SetModTime(name string, modTime time.Time) {
osPath := c.ToOSPath(name)
err := os.Chtimes(osPath, modTime, modTime)
if err != nil {
fs.Errorf(name, "Failed to set modification time of cached file: %v", err)
}
}
// cleanUp empties the cache of everything
func (c *cache) cleanUp() error {
// CleanUp empties the cache of everything
func (c *Cache) CleanUp() error {
return os.RemoveAll(c.root)
}
// walk walks the cache calling the function
func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error {
func (c *Cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error {
return filepath.Walk(c.root, func(osPath string, fi os.FileInfo, err error) error {
if err != nil {
return err
@ -419,7 +370,7 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error)
// updateStats walks the cache updating any atimes and sizes it finds
//
// it also updates used
func (c *cache) updateStats() error {
func (c *Cache) updateStats() error {
var newUsed int64
err := c.walk(func(osPath string, fi os.FileInfo, name string) error {
if !fi.IsDir() {
@ -439,11 +390,11 @@ func (c *cache) updateStats() error {
}
// purgeOld gets rid of any files that are over age
func (c *cache) purgeOld(maxAge time.Duration) {
c._purgeOld(maxAge, c.remove)
func (c *Cache) purgeOld(maxAge time.Duration) {
c._purgeOld(maxAge, c.Remove)
}
func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
func (c *Cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
c.itemMu.Lock()
defer c.itemMu.Unlock()
cutoff := time.Now().Add(-maxAge)
@ -462,11 +413,11 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
}
// Purge any empty directories
func (c *cache) purgeEmptyDirs() {
func (c *Cache) purgeEmptyDirs() {
c._purgeEmptyDirs(c.removeDir)
}
func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) {
func (c *Cache) _purgeEmptyDirs(removeDir func(name string) bool) {
c.itemMu.Lock()
defer c.itemMu.Unlock()
var dirs []string
@ -499,11 +450,11 @@ func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j
// Remove any files that are over quota starting from the
// oldest first
func (c *cache) purgeOverQuota(quota int64) {
c._purgeOverQuota(quota, c.remove)
func (c *Cache) purgeOverQuota(quota int64) {
c._purgeOverQuota(quota, c.Remove)
}
func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
func (c *Cache) _purgeOverQuota(quota int64, remove func(name string)) {
c.itemMu.Lock()
defer c.itemMu.Unlock()
@ -537,7 +488,7 @@ func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
}
// clean empties the cache of stuff if it can
func (c *cache) clean() {
func (c *Cache) clean() {
// Cache may be empty so end
_, err := os.Stat(c.root)
if os.IsNotExist(err) {
@ -575,7 +526,7 @@ func (c *cache) clean() {
// cleaner calls clean at regular intervals
//
// doesn't return until context is cancelled
func (c *cache) cleaner(ctx context.Context) {
func (c *Cache) cleaner(ctx context.Context) {
if c.opt.CachePollInterval <= 0 {
fs.Debugf(nil, "Cache cleaning thread disabled because poll interval <= 0")
return
@ -595,3 +546,50 @@ func (c *cache) cleaner(ctx context.Context) {
}
}
}
// copy an object to or from the remote while accounting for it
func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
if operations.NeedTransfer(context.TODO(), dst, src) {
newDst, err = operations.Copy(context.TODO(), f, dst, remote, src)
} else {
newDst = dst
}
return newDst, err
}
// Check the local file is up to date in the cache
func (c *Cache) Check(ctx context.Context, o fs.Object, remote string) error {
cacheObj, err := c.fcache.NewObject(ctx, remote)
if err == nil && cacheObj != nil {
_, err = copyObj(c.fcache, cacheObj, remote, o)
if err != nil {
return errors.Wrap(err, "failed to update cached file")
}
}
return nil
}
// Fetch fetches the object to the cache file
func (c *Cache) Fetch(ctx context.Context, o fs.Object, remote string) error {
_, err := copyObj(c.fcache, nil, remote, o)
return err
}
// Store stores the local cache file to the remote object, returning
// the new remote object. objOld is the old object if known.
func (c *Cache) Store(ctx context.Context, objOld fs.Object, remote string) (fs.Object, error) {
// Transfer the temp file to the remote
cacheObj, err := c.fcache.NewObject(ctx, remote)
if err != nil {
return nil, errors.Wrap(err, "failed to find cache file")
}
if objOld != nil {
remote = objOld.Remote() // use the path of the actual object if available
}
o, err := copyObj(c.fremote, objOld, remote, cacheObj)
if err != nil {
return nil, errors.Wrap(err, "failed to transfer file from cache to remote")
}
return o, nil
}

126
vfs/cache_test.go → vfs/vfscache/vfscache_test.go
View File

@ -1,4 +1,4 @@
package vfs
package vfscache
import (
"context"
@ -11,42 +11,20 @@ import (
"time"
"github.com/djherbis/times"
_ "github.com/rclone/rclone/backend/local" // import the local backend
"github.com/rclone/rclone/fstest"
"github.com/spf13/pflag"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Check CacheMode it satisfies the pflag interface
var _ pflag.Value = (*CacheMode)(nil)
func TestCacheModeString(t *testing.T) {
assert.Equal(t, "off", CacheModeOff.String())
assert.Equal(t, "full", CacheModeFull.String())
assert.Equal(t, "CacheMode(17)", CacheMode(17).String())
}
func TestCacheModeSet(t *testing.T) {
var m CacheMode
err := m.Set("full")
assert.NoError(t, err)
assert.Equal(t, CacheModeFull, m)
err = m.Set("potato")
assert.Error(t, err, "Unknown cache mode level")
err = m.Set("")
assert.Error(t, err, "Unknown cache mode level")
}
func TestCacheModeType(t *testing.T) {
var m CacheMode
assert.Equal(t, "CacheMode", m.Type())
// TestMain drives the tests
func TestMain(m *testing.M) {
fstest.TestMain(m)
}
// convert c.item to a string
func itemAsString(c *cache) []string {
func itemAsString(c *Cache) []string {
c.itemMu.Lock()
defer c.itemMu.Unlock()
var out []string
@ -65,17 +43,17 @@ func TestCacheNew(t *testing.T) {
defer cancel()
// Disable the cache cleaner as it interferes with these tests
opt := DefaultOpt
opt := vfscommon.DefaultOpt
opt.CachePollInterval = 0
c, err := newCache(ctx, r.Fremote, &opt)
c, err := New(ctx, r.Fremote, &opt)
require.NoError(t, err)
assert.Contains(t, c.root, "vfs")
assert.Contains(t, c.f.Root(), filepath.Base(r.Fremote.Root()))
assert.Contains(t, c.fcache.Root(), filepath.Base(r.Fremote.Root()))
assert.Equal(t, []string(nil), itemAsString(c))
// mkdir
p, err := c.mkdir("potato")
p, err := c.Mkdir("potato")
require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string{
@ -111,7 +89,7 @@ func TestCacheNew(t *testing.T) {
`name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0 size=0`,
}, itemAsString(c))
c.open("/potato")
c.Open("/potato")
assert.Equal(t, []string{
`name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1 size=0`,
@ -173,7 +151,7 @@ func TestCacheNew(t *testing.T) {
`name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1 size=6`,
}, itemAsString(c))
c.close("potato/")
c.Close("potato/")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0 size=5`,
@ -197,7 +175,7 @@ func TestCacheNew(t *testing.T) {
c.clean()
// cleanup
err = c.cleanUp()
err = c.CleanUp()
require.NoError(t, err)
_, err = os.Stat(c.root)
assert.True(t, os.IsNotExist(err))
@ -210,35 +188,35 @@ func TestCacheOpens(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c, err := newCache(ctx, r.Fremote, &DefaultOpt)
c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
require.NoError(t, err)
assert.Equal(t, []string(nil), itemAsString(c))
c.open("potato")
c.Open("potato")
assert.Equal(t, []string{
`name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c))
c.open("potato")
c.Open("potato")
assert.Equal(t, []string{
`name="" isFile=false opens=2 size=0`,
`name="potato" isFile=true opens=2 size=0`,
}, itemAsString(c))
c.close("potato")
c.Close("potato")
assert.Equal(t, []string{
`name="" isFile=false opens=1 size=0`,
`name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c))
c.close("potato")
c.Close("potato")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="potato" isFile=true opens=0 size=0`,
}, itemAsString(c))
c.open("potato")
c.open("a//b/c/d/one")
c.open("a/b/c/d/e/two")
c.open("a/b/c/d/e/f/three")
c.Open("potato")
c.Open("a//b/c/d/one")
c.Open("a/b/c/d/e/two")
c.Open("a/b/c/d/e/f/three")
assert.Equal(t, []string{
`name="" isFile=false opens=4 size=0`,
`name="a" isFile=false opens=3 size=0`,
@ -252,10 +230,10 @@ func TestCacheOpens(t *testing.T) {
`name="a/b/c/d/one" isFile=true opens=1 size=0`,
`name="potato" isFile=true opens=1 size=0`,
}, itemAsString(c))
c.close("potato")
c.close("a/b/c/d/one")
c.close("a/b/c/d/e/two")
c.close("a/b/c//d/e/f/three")
c.Close("potato")
c.Close("a/b/c/d/one")
c.Close("a/b/c/d/e/two")
c.Close("a/b/c//d/e/f/three")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
`name="a" isFile=false opens=0 size=0`,
@ -280,13 +258,13 @@ func TestCacheOpenMkdir(t *testing.T) {
defer cancel()
// Disable the cache cleaner as it interferes with these tests
opt := DefaultOpt
opt := vfscommon.DefaultOpt
opt.CachePollInterval = 0
c, err := newCache(ctx, r.Fremote, &opt)
c, err := New(ctx, r.Fremote, &opt)
require.NoError(t, err)
// open
c.open("sub/potato")
c.Open("sub/potato")
assert.Equal(t, []string{
`name="" isFile=false opens=1 size=0`,
@ -295,7 +273,7 @@ func TestCacheOpenMkdir(t *testing.T) {
}, itemAsString(c))
// mkdir
p, err := c.mkdir("sub/potato")
p, err := c.Mkdir("sub/potato")
require.NoError(t, err)
assert.Equal(t, "potato", filepath.Base(p))
assert.Equal(t, []string{
@ -318,7 +296,7 @@ func TestCacheOpenMkdir(t *testing.T) {
assert.True(t, fi.IsDir())
// close
c.close("sub/potato")
c.Close("sub/potato")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
@ -344,7 +322,7 @@ func TestCacheCacheDir(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c, err := newCache(ctx, r.Fremote, &DefaultOpt)
c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
require.NoError(t, err)
assert.Equal(t, []string(nil), itemAsString(c))
@ -379,7 +357,7 @@ func TestCachePurgeOld(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c, err := newCache(ctx, r.Fremote, &DefaultOpt)
c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
require.NoError(t, err)
// Test funcs
@ -400,10 +378,10 @@ func TestCachePurgeOld(t *testing.T) {
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string(nil), removed)
c.open("sub/dir2/potato2")
c.open("sub/dir/potato")
c.close("sub/dir2/potato2")
c.open("sub/dir/potato")
c.Open("sub/dir2/potato2")
c.Open("sub/dir/potato")
c.Close("sub/dir2/potato2")
c.Open("sub/dir/potato")
assert.Equal(t, []string{
`name="" isFile=false opens=2 size=0`,
@ -423,7 +401,7 @@ func TestCachePurgeOld(t *testing.T) {
"sub/dir2/",
}, removed)
c.close("sub/dir/potato")
c.Close("sub/dir/potato")
removed = nil
removedDir = true
@ -431,7 +409,7 @@ func TestCachePurgeOld(t *testing.T) {
c._purgeEmptyDirs(removeDir)
assert.Equal(t, []string(nil), removed)
c.close("sub/dir/potato")
c.Close("sub/dir/potato")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
@ -475,16 +453,16 @@ func TestCachePurgeOverQuota(t *testing.T) {
defer cancel()
// Disable the cache cleaner as it interferes with these tests
opt := DefaultOpt
opt := vfscommon.DefaultOpt
opt.CachePollInterval = 0
c, err := newCache(ctx, r.Fremote, &opt)
c, err := New(ctx, r.Fremote, &opt)
require.NoError(t, err)
// Test funcs
var removed []string
remove := func(name string) {
removed = append(removed, filepath.ToSlash(name))
c.remove(name)
c.Remove(name)
}
removed = nil
@ -500,14 +478,14 @@ func TestCachePurgeOverQuota(t *testing.T) {
assert.Equal(t, []string(nil), removed)
// Make some test files
c.open("sub/dir/potato")
p, err := c.mkdir("sub/dir/potato")
c.Open("sub/dir/potato")
p, err := c.Mkdir("sub/dir/potato")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello"), 0600)
require.NoError(t, err)
p, err = c.mkdir("sub/dir2/potato2")
c.open("sub/dir2/potato2")
p, err = c.Mkdir("sub/dir2/potato2")
c.Open("sub/dir2/potato2")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello2"), 0600)
require.NoError(t, err)
@ -527,8 +505,8 @@ func TestCachePurgeOverQuota(t *testing.T) {
assert.Equal(t, []string(nil), removed)
// Close the files
c.close("sub/dir/potato")
c.close("sub/dir2/potato2")
c.Close("sub/dir/potato")
c.Close("sub/dir2/potato2")
assert.Equal(t, []string{
`name="" isFile=false opens=0 size=0`,
@ -565,12 +543,12 @@ func TestCachePurgeOverQuota(t *testing.T) {
}, itemAsString(c))
// Put potato back
c.open("sub/dir/potato")
p, err = c.mkdir("sub/dir/potato")
c.Open("sub/dir/potato")
p, err = c.Mkdir("sub/dir/potato")
require.NoError(t, err)
err = ioutil.WriteFile(p, []byte("hello"), 0600)
require.NoError(t, err)
c.close("sub/dir/potato")
c.Close("sub/dir/potato")
// Update the stats to read the total size
err = c.updateStats()

49
vfs/vfscommon/cachemode.go Normal file
View File

@ -0,0 +1,49 @@
package vfscommon
import (
"fmt"
"github.com/rclone/rclone/lib/errors"
)
// CacheMode controls the functionality of the cache
type CacheMode byte
// CacheMode options
const (
CacheModeOff CacheMode = iota // cache nothing - return errors for writes which can't be satisfied
CacheModeMinimal // cache only the minimum, eg read/write opens
CacheModeWrites // cache all files opened with write intent
CacheModeFull // cache all files opened in any mode
)
var cacheModeToString = []string{
CacheModeOff: "off",
CacheModeMinimal: "minimal",
CacheModeWrites: "writes",
CacheModeFull: "full",
}
// String turns a CacheMode into a string
func (l CacheMode) String() string {
if l >= CacheMode(len(cacheModeToString)) {
return fmt.Sprintf("CacheMode(%d)", l)
}
return cacheModeToString[l]
}
// Set a CacheMode
func (l *CacheMode) Set(s string) error {
for n, name := range cacheModeToString {
if s != "" && name == s {
*l = CacheMode(n)
return nil
}
}
return errors.Errorf("Unknown cache mode level %q", s)
}
// Type of the value
func (l *CacheMode) Type() string {
return "CacheMode"
}

36
vfs/vfscommon/cachemode_test.go Normal file
View File

@ -0,0 +1,36 @@
package vfscommon
import (
"testing"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
)
// Check CacheMode it satisfies the pflag interface
var _ pflag.Value = (*CacheMode)(nil)
func TestCacheModeString(t *testing.T) {
assert.Equal(t, "off", CacheModeOff.String())
assert.Equal(t, "full", CacheModeFull.String())
assert.Equal(t, "CacheMode(17)", CacheMode(17).String())
}
func TestCacheModeSet(t *testing.T) {
var m CacheMode
err := m.Set("full")
assert.NoError(t, err)
assert.Equal(t, CacheModeFull, m)
err = m.Set("potato")
assert.Error(t, err, "Unknown cache mode level")
err = m.Set("")
assert.Error(t, err, "Unknown cache mode level")
}
func TestCacheModeType(t *testing.T) {
var m CacheMode
assert.Equal(t, "CacheMode", m.Type())
}

57
vfs/vfscommon/options.go Normal file
View File

@ -0,0 +1,57 @@
package vfscommon
import (
"os"
"runtime"
"time"
"github.com/rclone/rclone/fs"
)
// Options is options for creating the vfs
type Options struct {
NoSeek bool // don't allow seeking if set
NoChecksum bool // don't check checksums if set
ReadOnly bool // if set VFS is read only
NoModTime bool // don't read mod times for files
DirCacheTime time.Duration // how long to consider directory listing cache valid
PollInterval time.Duration
Umask int
UID uint32
GID uint32
DirPerms os.FileMode
FilePerms os.FileMode
ChunkSize fs.SizeSuffix // if > 0 read files in chunks
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
CacheMode CacheMode
CacheMaxAge time.Duration
CacheMaxSize fs.SizeSuffix
CachePollInterval time.Duration
CaseInsensitive bool
WriteWait time.Duration // time to wait for in-sequence write
ReadWait time.Duration // time to wait for in-sequence read
}
// DefaultOpt is the default values uses for Opt
var DefaultOpt = Options{
NoModTime: false,
NoChecksum: false,
NoSeek: false,
DirCacheTime: 5 * 60 * time.Second,
PollInterval: time.Minute,
ReadOnly: false,
Umask: 0,
UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
GID: ^uint32(0), // overriden for non windows in mount_unix.go
DirPerms: os.FileMode(0777),
FilePerms: os.FileMode(0666),
CacheMode: CacheModeOff,
CacheMaxAge: 3600 * time.Second,
CachePollInterval: 60 * time.Second,
ChunkSize: 128 * fs.MebiByte,
ChunkSizeLimit: -1,
CacheMaxSize: -1,
CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
WriteWait: 1000 * time.Millisecond,
ReadWait: 5 * time.Millisecond,
}

12
vfs/vfscommon/path.go Normal file
View File

@ -0,0 +1,12 @@
package vfscommon
import "path/filepath"
// FindParent returns the parent directory of name, or "" for the root
func FindParent(name string) string {
parent := filepath.Dir(name)
if parent == "." || parent == "/" {
parent = ""
}
return parent
}

4
vfs/vfsflags/vfsflags.go
View File

@ -4,13 +4,13 @@ package vfsflags
import (
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/spf13/pflag"
)
// Options set by command line flags
var (
Opt = vfs.DefaultOpt
Opt = vfscommon.DefaultOpt
DirPerms = &FileMode{Mode: &Opt.DirPerms}
FilePerms = &FileMode{Mode: &Opt.FilePerms}
)

13
vfs/vfstest/fs.go
View File

@ -24,6 +24,7 @@ import (
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -45,11 +46,11 @@ var (
func RunTests(t *testing.T, useVFS bool, fn MountFn) {
mountFn = fn
flag.Parse()
cacheModes := []vfs.CacheMode{
vfs.CacheModeOff,
vfs.CacheModeMinimal,
vfs.CacheModeWrites,
vfs.CacheModeFull,
cacheModes := []vfscommon.CacheMode{
vfscommon.CacheModeOff,
vfscommon.CacheModeMinimal,
vfscommon.CacheModeWrites,
vfscommon.CacheModeFull,
}
run = newRun(useVFS)
for _, cacheMode := range cacheModes {
@ -218,7 +219,7 @@ func (r *Run) umount() {
}
// cacheMode flushes the VFS and changes the CacheMode
func (r *Run) cacheMode(cacheMode vfs.CacheMode) {
func (r *Run) cacheMode(cacheMode vfscommon.CacheMode) {
if r.skip {
log.Printf("FUSE not found so skipping cacheMode")
return

7
vfs/vfstest/write.go
View File

@ -5,10 +5,9 @@ import (
"runtime"
"testing"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/rclone/rclone/vfs"
)
// TestWriteFileNoWrite tests writing a file with no write()'s to it
@ -92,7 +91,7 @@ func TestWriteFileDup(t *testing.T) {
run.skipIfVFS(t)
run.skipIfNoFUSE(t)
if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
t.Skip("not supported on vfs-cache-mode < writes")
return
}
@ -137,7 +136,7 @@ func TestWriteFileDup(t *testing.T) {
func TestWriteFileAppend(t *testing.T) {
run.skipIfNoFUSE(t)
if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
t.Skip("not supported on vfs-cache-mode < writes")
return
}

5
vfs/vfstest/write_unix.go
View File

@ -6,10 +6,9 @@ import (
"runtime"
"testing"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"golang.org/x/sys/unix"
"github.com/rclone/rclone/vfs"
)
// TestWriteFileDoubleClose tests double close on write
@ -46,7 +45,7 @@ func TestWriteFileDoubleClose(t *testing.T) {
// write to the other dup
_, err = unix.Write(fd2, buf)
if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
// produces an error if cache mode < writes
assert.Error(t, err, "input/output error")
} else {