wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-27 10:01:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: support option for cache streamed chunk data into temp file for potential retry.

Browse Source
This commit is contained in:
HFO4
2022-03-26 15:33:31 +08:00
parent 636ac52a3f
commit 31315c86ee
6 changed files with 73 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/filesystem/driver/onedrive/api.go
View File

@@ -221,8 +221,16 @@ func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL stri
return &uploadSession, nil
}
var index = 0
// UploadChunk 上传分片
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
index++
if index == 1 || index == 2 {
request.BlackHole(content)
return nil, errors.New("error")
}
res, err := client.request(
ctx, "PUT", uploadURL, content,
request.WithContentLength(current.Length()),
@@ -281,7 +289,7 @@ func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
})
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := client.UploadChunk(ctx, uploadURL, content, current)

4
pkg/filesystem/driver/oss/handler.go
View File

@@ -252,7 +252,7 @@ func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
})
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
@@ -435,7 +435,7 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
uploadSession.UploadID = imur.UploadID
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{})
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {

2
pkg/filesystem/driver/remote/client.go
View File

@@ -92,7 +92,7 @@ func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
})
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())

2
pkg/filesystem/driver/s3/handler.go
View File

@@ -342,7 +342,7 @@ func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *seri
uploadSession.UploadID = *res.UploadId
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{})
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {