wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-27 01:51:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: task queue / compression task

Browse Source
This commit is contained in:
HFO4
2020-02-02 14:40:07 +08:00
parent b1490a665c
commit e722c33cd5
25 changed files with 573 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/filesystem/archive.go
View File

@@ -20,7 +20,7 @@ import (
*/
// Compress 创建给定目录和文件的压缩文件
func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint) (string, error) {
func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint, isArchive bool) (string, error) {
// 查找待压缩目录
folders, err := model.GetFoldersByIDs(folderIDs, fs.User.ID)
if err != nil && len(folders) != 0 {
@@ -66,8 +66,13 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint) (
}
// 创建临时压缩文件
saveFolder := "archive"
if !isArchive {
saveFolder = "compress"
}
zipFilePath := filepath.Join(
model.GetSettingByName("temp_path"),
saveFolder,
fmt.Sprintf("archive_%d.zip", time.Now().UnixNano()),
)
zipFile, err := util.CreatNestedFile(zipFilePath)
@@ -91,7 +96,7 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint) (
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
return "", ErrClientCanceled
default:
fs.doCompress(ctx, nil, &folders[i], zipWriter, true)
fs.doCompress(ctx, nil, &folders[i], zipWriter, isArchive)
}
}
@@ -102,7 +107,7 @@ func (fs *FileSystem) Compress(ctx context.Context, folderIDs, fileIDs []uint) (
fs.cancelCompress(ctx, zipWriter, zipFile, zipFilePath)
return "", ErrClientCanceled
default:
fs.doCompress(ctx, &files[i], nil, zipWriter, true)
fs.doCompress(ctx, &files[i], nil, zipWriter, isArchive)
}
}

6
pkg/filesystem/archive_test.go
View File

@@ -51,7 +51,7 @@ func TestFileSystem_Compress(t *testing.T) {
// 查找上传策略
asserts.NoError(cache.Set("policy_1", model.Policy{Type: "local"}, -1))
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1})
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
asserts.NotEmpty(zipFile)
asserts.Contains(zipFile, "archive_")
@@ -76,7 +76,7 @@ func TestFileSystem_Compress(t *testing.T) {
)
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1})
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.Error(err)
asserts.Empty(zipFile)
}
@@ -100,7 +100,7 @@ func TestFileSystem_Compress(t *testing.T) {
)
asserts.NoError(cache.Set("setting_temp_path", "tests", -1))
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1})
zipFile, err := fs.Compress(ctx, []uint{1}, []uint{1}, true)
asserts.Error(err)
asserts.Equal(ErrObjectNotExist, err)
asserts.Empty(zipFile)

190
pkg/filesystem/image_test.go
View File

@@ -1,190 +0,0 @@
package filesystem
import (
"context"
"errors"
"fmt"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"image"
"image/jpeg"
"os"
"testing"
)
func CreateTestImage() *os.File {
file, err := os.Create("TestFileSystem_GenerateThumbnail.jpeg")
alpha := image.NewAlpha(image.Rect(0, 0, 500, 200))
jpeg.Encode(file, alpha, nil)
if err != nil {
fmt.Println(err)
}
_, _ = file.Seek(0, 0)
return file
}
func TestFileSystem_GetThumb(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
cache.Set("setting_preview_timeout", "60", 0)
// 正常
{
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
}
testHandler := new(FileHeaderMock)
testHandler.On("Thumb", testMock.Anything, "123.jpg").Return(&response.ContentResponse{URL: "123"}, nil)
fs.Handler = testHandler
mock.ExpectQuery("SELECT(.+)").
WithArgs(10, 1).
WillReturnRows(
sqlmock.NewRows(
[]string{"id", "pic_info", "source_name", "policy_id"}).
AddRow(10, "10,10", "123.jpg", 154),
)
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows(
[]string{"id", "type"}).
AddRow(154, "mock"),
)
res, err := fs.GetThumb(ctx, 10)
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.NoError(err)
asserts.Equal("123", res.URL)
}
// 文件不存在
{
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
}
mock.ExpectQuery("SELECT(.+)").
WithArgs(10, 1).
WillReturnRows(
sqlmock.NewRows(
[]string{"id", "pic_info", "source_name"}),
)
_, err := fs.GetThumb(ctx, 10)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
}
func TestFileSystem_GenerateThumbnail(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
}
ctx := context.Background()
// 成功
{
src := CreateTestImage()
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "TestFileSystem_GenerateThumbnail.jpeg").Return(src, nil)
fs.Handler = testHandler
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
file := &model.File{
Model: gorm.Model{ID: 1},
Name: "123.jpg",
SourceName: "TestFileSystem_GenerateThumbnail.jpeg",
}
fs.GenerateThumbnail(ctx, file)
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.True(util.Exists("TestFileSystem_GenerateThumbnail.jpeg" + conf.ThumbConfig.FileSuffix))
}
// 成功,不进行数据库更新
{
src := CreateTestImage()
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "TestFileSystem_GenerateThumbnail.jpeg").Return(src, nil)
fs.Handler = testHandler
file := &model.File{
Name: "123.jpg",
SourceName: "TestFileSystem_GenerateThumbnail.jpeg",
}
fs.GenerateThumbnail(ctx, file)
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
asserts.True(util.Exists("TestFileSystem_GenerateThumbnail.jpeg" + conf.ThumbConfig.FileSuffix))
}
// 更新信息失败后删除文件
{
src := CreateTestImage()
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "TestFileSystem_GenerateThumbnail.jpeg").Return(src, nil)
testHandler.On("Delete", testMock.Anything, testMock.Anything).Return([]string{}, nil)
fs.Handler = testHandler
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
file := &model.File{
Model: gorm.Model{ID: 1},
Name: "123.jpg",
SourceName: "TestFileSystem_GenerateThumbnail.jpeg",
}
fs.GenerateThumbnail(ctx, file)
asserts.NoError(mock.ExpectationsWereMet())
testHandler.AssertExpectations(t)
}
// 不能生成缩略图
{
file := &model.File{
Model: gorm.Model{ID: 1},
Name: "123.123",
SourceName: "TestFileSystem_GenerateThumbnail.jpeg",
}
fs.GenerateThumbnail(ctx, file)
asserts.NoError(mock.ExpectationsWereMet())
}
}
func TestFileSystem_GenerateThumbnailSize(t *testing.T) {
asserts := assert.New(t)
fs := FileSystem{
User: &model.User{
Model: gorm.Model{ID: 1},
},
}
asserts.NotPanics(func() {
_, _ = fs.GenerateThumbnailSize(0, 0)
})
}

44
pkg/filesystem/upload.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-gonic/gin"
"os"
"path"
)
@@ -114,8 +115,10 @@ func (fs *FileSystem) CancelUpload(ctx context.Context, path string, file FileHe
var reqContext context.Context
if ginCtx, ok := ctx.Value(fsctx.GinCtx).(*gin.Context); ok {
reqContext = ginCtx.Request.Context()
} else if reqCtx, ok := ctx.Value(fsctx.HTTPCtx).(context.Context); ok {
reqContext = reqCtx
} else {
reqContext = ctx.Value(fsctx.HTTPCtx).(context.Context)
return
}
select {
@@ -182,3 +185,42 @@ func (fs *FileSystem) GetUploadToken(ctx context.Context, path string, size uint
return &credential, nil
}
// UploadFromPath 将本机已有文件上传到用户的文件系统
func (fs *FileSystem) UploadFromPath(ctx context.Context, src, dst string) error {
file, err := os.Open(src)
if err != nil {
return err
}
defer file.Close()
// 获取源文件大小
fi, err := file.Stat()
if err != nil {
return err
}
size := fi.Size()
// 构建文件头
fileName := path.Base(dst)
filePath := path.Dir(dst)
fileData := local.FileStream{
File: file,
Size: uint64(size),
Name: fileName,
VirtualPath: filePath,
}
// 给文件系统分配钩子
fs.Use("BeforeUpload", HookValidateFile)
fs.Use("BeforeUpload", HookValidateCapacity)
fs.Use("AfterUploadCanceled", HookDeleteTempFile)
fs.Use("AfterUploadCanceled", HookGiveBackCapacity)
fs.Use("AfterUpload", GenericAfterUpload)
fs.Use("AfterValidateFailed", HookDeleteTempFile)
fs.Use("AfterValidateFailed", HookGiveBackCapacity)
fs.Use("AfterUploadFailed", HookGiveBackCapacity)
// 开始上传
return fs.Upload(ctx, fileData)
}