local: speed up multi thread downloads by using sparse files on Windows

Before this change rclone didn't use sparse files on Windows. This
means that when you downloaded a file with multithread download it
wrote the entire file with zeros first on the first write not at the
start of the file.

This change makes the file be sparse on Windows. Linux/macOS files
were already sparse.
master
Nick Craig-Wood 2020-03-06 12:41:48 +00:00
parent dfc7215bf9
commit a492c0fb0e
4 changed files with 29 additions and 0 deletions

View File

@ -1068,6 +1068,12 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err)
}
// Set the file to be a sparse file (important on Windows)
err = setSparse(out)
if err != nil {
fs.Debugf(o, "Failed to set sparse: %v", err)
}
return out, nil
}

View File

@ -8,3 +8,8 @@ import "os"
func preAllocate(size int64, out *os.File) error {
return nil
}
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
return nil
}

View File

@ -44,3 +44,8 @@ again:
// }
return err
}
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
return nil
}

View File

@ -77,3 +77,16 @@ func preAllocate(size int64, out *os.File) error {
return nil
}
const (
FSCTL_SET_SPARSE = 0x000900c4
)
// setSparse makes the file be a sparse file
func setSparse(out *os.File) error {
err := syscall.DeviceIoControl(syscall.Handle(out.Fd()), FSCTL_SET_SPARSE, nil, 0, nil, 0, nil, nil)
if err != nil {
return errors.Wrap(err, "DeviceIoControl FSCTL_SET_SPARSE")
}
return nil
}