diff --git a/backend/swift/swift.go b/backend/swift/swift.go index b0431aeba..7c1cfbe64 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -25,6 +25,7 @@ import ( "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/readers" ) // Constants @@ -1224,8 +1225,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } o.headers = nil // wipe old metadata } else { + var inCount *readers.CountingReader if size >= 0 { headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length if we know it + } else { + // otherwise count the size for later + inCount = readers.NewCountingReader(in) + in = inCount } var rxHeaders swift.Headers err = o.fs.pacer.CallNoRetry(func() (bool, error) { @@ -1242,6 +1248,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op o.md5 = rxHeaders["ETag"] o.contentType = contentType o.headers = headers + if inCount != nil { + // update the size if streaming from the reader + o.size = int64(inCount.BytesRead()) + } } // If file was a dynamic large object then remove old/all segments diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.go index 7b182230a..f7f4e0ecb 100644 --- a/backend/swift/swift_test.go +++ b/backend/swift/swift_test.go @@ -2,10 +2,19 @@ package swift import ( + "bytes" + "context" + "io" "testing" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestIntegration runs integration tests against the remote @@ -21,3 +30,50 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) { } var _ fstests.SetUploadChunkSizer = (*Fs)(nil) + +// Check that PutStream works with NoChunk as it is the major code +// deviation +func (f *Fs) testNoChunk(t *testing.T) { + ctx := context.Background() + f.opt.NoChunk = true + defer func() { + f.opt.NoChunk = false + }() + + file := fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "piped data no chunk.txt", + Size: -1, // use unknown size during upload + } + + const contentSize = 100 + + contents := random.String(contentSize) + buf := bytes.NewBufferString(contents) + uploadHash := hash.NewMultiHasher() + in := io.TeeReader(buf, uploadHash) + + file.Size = -1 + obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) + obj, err := f.Features().PutStream(ctx, in, obji) + require.NoError(t, err) + + file.Hashes = uploadHash.Sums() + file.Size = int64(contentSize) // use correct size when checking + file.Check(t, obj, f.Precision()) + + // Re-read the object and check again + obj, err = f.NewObject(ctx, file.Path) + require.NoError(t, err) + file.Check(t, obj, f.Precision()) + + // Delete the object + assert.NoError(t, obj.Remove(ctx)) +} + +// Additional tests that aren't in the framework +func (f *Fs) InternalTest(t *testing.T) { + t.Run("NoChunk", f.testNoChunk) +} + +var _ fstests.InternalTester = (*Fs)(nil)