rclone/vfs/read_write_test.go

743 lines
18 KiB
Go

package vfs
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"time"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func cleanup(t *testing.T, r *fstest.Run, vfs *VFS) {
assert.NoError(t, vfs.CleanUp())
vfs.Shutdown()
r.Finalise()
}
// Create a file and open it with the flags passed in
func rwHandleCreateFlags(t *testing.T, r *fstest.Run, create bool, filename string, flags int) (*VFS, *RWFileHandle) {
opt := DefaultOpt
opt.CacheMode = CacheModeFull
vfs := New(r.Fremote, &opt)
if create {
file1 := r.WriteObject(context.Background(), filename, "0123456789abcdef", t1)
fstest.CheckItems(t, r.Fremote, file1)
}
h, err := vfs.OpenFile(filename, flags, 0777)
require.NoError(t, err)
fh, ok := h.(*RWFileHandle)
require.True(t, ok)
return vfs, fh
}
// Open a file for read
func rwHandleCreateReadOnly(t *testing.T, r *fstest.Run) (*VFS, *RWFileHandle) {
return rwHandleCreateFlags(t, r, true, "dir/file1", os.O_RDONLY)
}
// Open a file for write
func rwHandleCreateWriteOnly(t *testing.T, r *fstest.Run) (*VFS, *RWFileHandle) {
return rwHandleCreateFlags(t, r, false, "file1", os.O_WRONLY|os.O_CREATE)
}
// read data from the string
func rwReadString(t *testing.T, fh *RWFileHandle, n int) string {
buf := make([]byte, n)
n, err := fh.Read(buf)
if err != io.EOF {
assert.NoError(t, err)
}
return string(buf[:n])
}
func TestRWFileHandleMethodsRead(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateReadOnly(t, r)
defer cleanup(t, r, vfs)
// String
assert.Equal(t, "dir/file1 (rw)", fh.String())
assert.Equal(t, "<nil *RWFileHandle>", (*RWFileHandle)(nil).String())
assert.Equal(t, "<nil *RWFileHandle.file>", new(RWFileHandle).String())
// Node
node := fh.Node()
assert.Equal(t, "file1", node.Name())
// Size
assert.Equal(t, int64(16), fh.Size())
// No opens yet
assert.Equal(t, 0, fh.file.rwOpens())
// Read 1
assert.Equal(t, "0", rwReadString(t, fh, 1))
// Open after the read
assert.Equal(t, 1, fh.file.rwOpens())
// Read remainder
assert.Equal(t, "123456789abcdef", rwReadString(t, fh, 256))
// Read EOF
buf := make([]byte, 16)
_, err := fh.Read(buf)
assert.Equal(t, io.EOF, err)
// Sync
err = fh.Sync()
assert.NoError(t, err)
// Stat
var fi os.FileInfo
fi, err = fh.Stat()
assert.NoError(t, err)
assert.Equal(t, int64(16), fi.Size())
assert.Equal(t, "file1", fi.Name())
// Close
assert.False(t, fh.closed)
assert.Equal(t, nil, fh.Close())
assert.True(t, fh.closed)
// No opens again
assert.Equal(t, 0, fh.file.rwOpens())
// Close again
assert.Equal(t, ECLOSED, fh.Close())
}
func TestRWFileHandleSeek(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateReadOnly(t, r)
defer cleanup(t, r, vfs)
assert.Equal(t, fh.opened, false)
// Check null seeks don't open the file
n, err := fh.Seek(0, io.SeekStart)
assert.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.Equal(t, fh.opened, false)
n, err = fh.Seek(0, io.SeekCurrent)
assert.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.Equal(t, fh.opened, false)
assert.Equal(t, "0", rwReadString(t, fh, 1))
// 0 means relative to the origin of the file,
n, err = fh.Seek(5, io.SeekStart)
assert.NoError(t, err)
assert.Equal(t, int64(5), n)
assert.Equal(t, "5", rwReadString(t, fh, 1))
// 1 means relative to the current offset
n, err = fh.Seek(-3, io.SeekCurrent)
assert.NoError(t, err)
assert.Equal(t, int64(3), n)
assert.Equal(t, "3", rwReadString(t, fh, 1))
// 2 means relative to the end.
n, err = fh.Seek(-3, io.SeekEnd)
assert.NoError(t, err)
assert.Equal(t, int64(13), n)
assert.Equal(t, "d", rwReadString(t, fh, 1))
// Seek off the end
_, err = fh.Seek(100, io.SeekStart)
assert.NoError(t, err)
// Get the error on read
buf := make([]byte, 16)
l, err := fh.Read(buf)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, l)
// Close
assert.Equal(t, nil, fh.Close())
}
func TestRWFileHandleReadAt(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateReadOnly(t, r)
defer cleanup(t, r, vfs)
// read from start
buf := make([]byte, 1)
n, err := fh.ReadAt(buf, 0)
require.NoError(t, err)
assert.Equal(t, 1, n)
assert.Equal(t, "0", string(buf[:n]))
// seek forwards
n, err = fh.ReadAt(buf, 5)
require.NoError(t, err)
assert.Equal(t, 1, n)
assert.Equal(t, "5", string(buf[:n]))
// seek backwards
n, err = fh.ReadAt(buf, 1)
require.NoError(t, err)
assert.Equal(t, 1, n)
assert.Equal(t, "1", string(buf[:n]))
// read exactly to the end
buf = make([]byte, 6)
n, err = fh.ReadAt(buf, 10)
require.NoError(t, err)
assert.Equal(t, 6, n)
assert.Equal(t, "abcdef", string(buf[:n]))
// read off the end
buf = make([]byte, 256)
n, err = fh.ReadAt(buf, 10)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 6, n)
assert.Equal(t, "abcdef", string(buf[:n]))
// read starting off the end
n, err = fh.ReadAt(buf, 100)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
// Properly close the file
assert.NoError(t, fh.Close())
// check reading on closed file
_, err = fh.ReadAt(buf, 100)
assert.Equal(t, ECLOSED, err)
}
func TestRWFileHandleFlushRead(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateReadOnly(t, r)
defer cleanup(t, r, vfs)
// Check Flush does nothing if read not called
err := fh.Flush()
assert.NoError(t, err)
assert.False(t, fh.closed)
// Read data
buf := make([]byte, 256)
n, err := fh.Read(buf)
assert.True(t, err == io.EOF || err == nil)
assert.Equal(t, 16, n)
// Check Flush does nothing if read called
err = fh.Flush()
assert.NoError(t, err)
assert.False(t, fh.closed)
// Check flush does nothing if called again
err = fh.Flush()
assert.NoError(t, err)
assert.False(t, fh.closed)
// Properly close the file
assert.NoError(t, fh.Close())
}
func TestRWFileHandleReleaseRead(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateReadOnly(t, r)
defer cleanup(t, r, vfs)
// Read data
buf := make([]byte, 256)
n, err := fh.Read(buf)
assert.True(t, err == io.EOF || err == nil)
assert.Equal(t, 16, n)
// Check Release closes file
err = fh.Release()
assert.NoError(t, err)
assert.True(t, fh.closed)
// Check Release does nothing if called again
err = fh.Release()
assert.NoError(t, err)
assert.True(t, fh.closed)
}
/// ------------------------------------------------------------
func TestRWFileHandleMethodsWrite(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateWriteOnly(t, r)
defer cleanup(t, r, vfs)
// 1 opens since we opened with O_CREATE and the file didn't
// exist in the cache
assert.Equal(t, 1, fh.file.rwOpens())
// String
assert.Equal(t, "file1 (rw)", fh.String())
assert.Equal(t, "<nil *RWFileHandle>", (*RWFileHandle)(nil).String())
assert.Equal(t, "<nil *RWFileHandle.file>", new(RWFileHandle).String())
// Node
node := fh.Node()
assert.Equal(t, "file1", node.Name())
offset := func() int64 {
n, err := fh.Seek(0, io.SeekCurrent)
require.NoError(t, err)
return n
}
// Offset #1
assert.Equal(t, int64(0), offset())
assert.Equal(t, int64(0), node.Size())
// Size #1
assert.Equal(t, int64(0), fh.Size())
// Write
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// Open after the write
assert.Equal(t, 1, fh.file.rwOpens())
// Offset #2
assert.Equal(t, int64(5), offset())
assert.Equal(t, int64(5), node.Size())
// Size #2
assert.Equal(t, int64(5), fh.Size())
// WriteString
n, err = fh.WriteString(" world!")
assert.NoError(t, err)
assert.Equal(t, 7, n)
// Sync
err = fh.Sync()
assert.NoError(t, err)
// Stat
var fi os.FileInfo
fi, err = fh.Stat()
assert.NoError(t, err)
assert.Equal(t, int64(12), fi.Size())
assert.Equal(t, "file1", fi.Name())
// Truncate
err = fh.Truncate(11)
assert.NoError(t, err)
// Close
assert.NoError(t, fh.Close())
// No opens again
assert.Equal(t, 0, fh.file.rwOpens())
// Check double close
err = fh.Close()
assert.Equal(t, ECLOSED, err)
// check vfs
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string{"file1,11,false"})
// check the underlying r.Fremote but not the modtime
file1 := fstest.NewItem("file1", "hello world", t1)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{}, fs.ModTimeNotSupported)
}
func TestRWFileHandleWriteAt(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateWriteOnly(t, r)
defer cleanup(t, r, vfs)
offset := func() int64 {
n, err := fh.Seek(0, io.SeekCurrent)
require.NoError(t, err)
return n
}
// Preconditions
assert.Equal(t, int64(0), offset())
assert.True(t, fh.opened)
assert.False(t, fh.writeCalled)
assert.True(t, fh.changed)
// Write the data
n, err := fh.WriteAt([]byte("hello**"), 0)
assert.NoError(t, err)
assert.Equal(t, 7, n)
// After write
assert.Equal(t, int64(0), offset())
assert.True(t, fh.writeCalled)
// Write more data
n, err = fh.WriteAt([]byte(" world"), 5)
assert.NoError(t, err)
assert.Equal(t, 6, n)
// Close
assert.NoError(t, fh.Close())
// Check can't write on closed handle
n, err = fh.WriteAt([]byte("hello"), 0)
assert.Equal(t, ECLOSED, err)
assert.Equal(t, 0, n)
// check vfs
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string{"file1,11,false"})
// check the underlying r.Fremote but not the modtime
file1 := fstest.NewItem("file1", "hello world", t1)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{}, fs.ModTimeNotSupported)
}
func TestRWFileHandleWriteNoWrite(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateWriteOnly(t, r)
defer cleanup(t, r, vfs)
// Close the file without writing to it
err := fh.Close()
if errors.Cause(err) == fs.ErrorCantUploadEmptyFiles {
t.Logf("skipping test: %v", err)
return
}
assert.NoError(t, err)
// Create a different file (not in the cache)
h, err := vfs.OpenFile("file2", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
require.NoError(t, err)
// Close it with Flush and Release
err = h.Flush()
assert.NoError(t, err)
err = h.Release()
assert.NoError(t, err)
// check vfs
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string{"file1,0,false", "file2,0,false"})
// check the underlying r.Fremote but not the modtime
file1 := fstest.NewItem("file1", "", t1)
file2 := fstest.NewItem("file2", "", t1)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, []string{}, fs.ModTimeNotSupported)
}
func TestRWFileHandleFlushWrite(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateWriteOnly(t, r)
defer cleanup(t, r, vfs)
// Check that the file has been create and is open
assert.True(t, fh.opened)
// Write some data
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// Check Flush does not close file if write called
err = fh.Flush()
assert.NoError(t, err)
assert.False(t, fh.closed)
// Check flush does nothing if called again
err = fh.Flush()
assert.NoError(t, err)
assert.False(t, fh.closed)
// Check that Close closes the file
err = fh.Close()
assert.NoError(t, err)
assert.True(t, fh.closed)
}
func TestRWFileHandleReleaseWrite(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateWriteOnly(t, r)
defer cleanup(t, r, vfs)
// Write some data
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// Check Release closes file
err = fh.Release()
assert.NoError(t, err)
assert.True(t, fh.closed)
// Check Release does nothing if called again
err = fh.Release()
assert.NoError(t, err)
assert.True(t, fh.closed)
}
// check the size of the file through the open file (if not nil) and via stat
func assertSize(t *testing.T, vfs *VFS, fh *RWFileHandle, filepath string, size int64) {
if fh != nil {
assert.Equal(t, size, fh.Size())
}
fi, err := vfs.Stat(filepath)
require.NoError(t, err)
assert.Equal(t, size, fi.Size())
}
func TestRWFileHandleSizeTruncateExisting(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateFlags(t, r, true, "dir/file1", os.O_WRONLY|os.O_TRUNC)
defer cleanup(t, r, vfs)
// check initial size after opening
assertSize(t, vfs, fh, "dir/file1", 0)
// write some bytes
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// check size after writing
assertSize(t, vfs, fh, "dir/file1", 5)
// close
assert.NoError(t, fh.Close())
// check size after close
assertSize(t, vfs, nil, "dir/file1", 5)
}
func TestRWFileHandleSizeCreateExisting(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateFlags(t, r, true, "dir/file1", os.O_WRONLY|os.O_CREATE)
defer cleanup(t, r, vfs)
// check initial size after opening
assertSize(t, vfs, fh, "dir/file1", 16)
// write some bytes
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// check size after writing
assertSize(t, vfs, fh, "dir/file1", 16)
// write some more bytes
n, err = fh.Write([]byte("helloHELLOhello"))
assert.NoError(t, err)
assert.Equal(t, 15, n)
// check size after writing
assertSize(t, vfs, fh, "dir/file1", 20)
// close
assert.NoError(t, fh.Close())
// check size after close
assertSize(t, vfs, nil, "dir/file1", 20)
}
func TestRWFileHandleSizeCreateNew(t *testing.T) {
r := fstest.NewRun(t)
vfs, fh := rwHandleCreateFlags(t, r, false, "file1", os.O_WRONLY|os.O_CREATE)
defer cleanup(t, r, vfs)
// check initial size after opening
assertSize(t, vfs, fh, "file1", 0)
// write some bytes
n, err := fh.Write([]byte("hello"))
assert.NoError(t, err)
assert.Equal(t, 5, n)
// check size after writing
assertSize(t, vfs, fh, "file1", 5)
// check size after writing
assertSize(t, vfs, fh, "file1", 5)
// close
assert.NoError(t, fh.Close())
// check size after close
assertSize(t, vfs, nil, "file1", 5)
}
func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
fileName := "open-test-file"
// first try with file not existing
_, err := vfs.Stat(fileName)
require.True(t, os.IsNotExist(err), test.what)
f, openNonExistentErr := vfs.OpenFile(fileName, test.flags, 0666)
var readNonExistentErr error
var writeNonExistentErr error
if openNonExistentErr == nil {
// read some bytes
buf := []byte{0, 0}
_, readNonExistentErr = f.Read(buf)
// write some bytes
_, writeNonExistentErr = f.Write([]byte("hello"))
// close
err = f.Close()
require.NoError(t, err, test.what)
}
// write the file
f, err = vfs.OpenFile(fileName, os.O_WRONLY|os.O_CREATE, 0777)
require.NoError(t, err, test.what)
_, err = f.Write([]byte("hello"))
require.NoError(t, err, test.what)
err = f.Close()
require.NoError(t, err, test.what)
// then open file and try with file existing
f, openExistingErr := vfs.OpenFile(fileName, test.flags, 0666)
var readExistingErr error
var writeExistingErr error
if openExistingErr == nil {
// read some bytes
buf := []byte{0, 0}
_, readExistingErr = f.Read(buf)
// write some bytes
_, writeExistingErr = f.Write([]byte("HEL"))
// close
err = f.Close()
require.NoError(t, err, test.what)
}
// read the file
f, err = vfs.OpenFile(fileName, os.O_RDONLY, 0)
require.NoError(t, err, test.what)
buf, err := ioutil.ReadAll(f)
require.NoError(t, err, test.what)
err = f.Close()
require.NoError(t, err, test.what)
contents := string(buf)
// remove file
node, err := vfs.Stat(fileName)
require.NoError(t, err, test.what)
err = node.Remove()
require.NoError(t, err, test.what)
// check
assert.Equal(t, test.openNonExistentErr, openNonExistentErr, "openNonExistentErr: %s: want=%v, got=%v", test.what, test.openNonExistentErr, openNonExistentErr)
assert.Equal(t, test.readNonExistentErr, readNonExistentErr, "readNonExistentErr: %s: want=%v, got=%v", test.what, test.readNonExistentErr, readNonExistentErr)
assert.Equal(t, test.writeNonExistentErr, writeNonExistentErr, "writeNonExistentErr: %s: want=%v, got=%v", test.what, test.writeNonExistentErr, writeNonExistentErr)
assert.Equal(t, test.openExistingErr, openExistingErr, "openExistingErr: %s: want=%v, got=%v", test.what, test.openExistingErr, openExistingErr)
assert.Equal(t, test.readExistingErr, readExistingErr, "readExistingErr: %s: want=%v, got=%v", test.what, test.readExistingErr, readExistingErr)
assert.Equal(t, test.writeExistingErr, writeExistingErr, "writeExistingErr: %s: want=%v, got=%v", test.what, test.writeExistingErr, writeExistingErr)
assert.Equal(t, test.contents, contents, test.what)
}
func TestRWFileHandleOpenTests(t *testing.T) {
r := fstest.NewRun(t)
opt := DefaultOpt
opt.CacheMode = CacheModeFull
vfs := New(r.Fremote, &opt)
defer cleanup(t, r, vfs)
for _, test := range openTests {
testRWFileHandleOpenTest(t, vfs, &test)
}
}
// tests mod time on open files
func TestRWFileModTimeWithOpenWriters(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if !canSetModTime(t, r) {
return
}
vfs, fh := rwHandleCreateWriteOnly(t, r)
mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
_, err := fh.Write([]byte{104, 105})
require.NoError(t, err)
err = fh.Node().SetModTime(mtime)
require.NoError(t, err)
// Using Flush/Release to mimic mount instead of Close
err = fh.Flush()
require.NoError(t, err)
err = fh.Release()
require.NoError(t, err)
info, err := vfs.Stat("file1")
require.NoError(t, err)
if r.Fremote.Precision() != fs.ModTimeNotSupported {
// avoid errors because of timezone differences
assert.Equal(t, info.ModTime().Unix(), mtime.Unix(), fmt.Sprintf("Time mismatch: %v != %v", info.ModTime(), mtime))
}
file1 := fstest.NewItem("file1", "hi", mtime)
fstest.CheckItems(t, r.Fremote, file1)
}
func TestRWCacheRename(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if !operations.CanServerSideMove(r.Fremote) {
t.Skip("skip as can't rename files")
}
opt := DefaultOpt
opt.CacheMode = CacheModeFull
vfs := New(r.Fremote, &opt)
h, err := vfs.OpenFile("rename_me", os.O_WRONLY|os.O_CREATE, 0777)
require.NoError(t, err)
_, err = h.WriteString("hello")
require.NoError(t, err)
fh, ok := h.(*RWFileHandle)
require.True(t, ok)
err = fh.Sync()
require.NoError(t, err)
err = fh.Close()
require.NoError(t, err)
assert.True(t, vfs.cache.exists("rename_me"))
err = vfs.Rename("rename_me", "i_was_renamed")
require.NoError(t, err)
assert.False(t, vfs.cache.exists("rename_me"))
assert.True(t, vfs.cache.exists("i_was_renamed"))
}