wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-26 09:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(thumb): new thumb pipeline model to generate thumb on-demand

Browse Source
This commit is contained in:
Aaron Liu
2023-04-07 19:08:54 +08:00
parent da1eaf2d1f
commit f36e39991d
23 changed files with 308 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/filesystem/driver/cos/handler.go
View File

@@ -222,7 +222,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -234,7 +234,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
source, err := handler.signSourceURL(
ctx,
path,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
&urlOption{},
)

8
pkg/filesystem/driver/handler.go
View File

@@ -2,12 +2,18 @@ package driver
import (
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"net/url"
)
var (
ErrorThumbNotExist = fmt.Errorf("thumb not exist")
)
// Handler 存储策略适配器
type Handler interface {
// 上传文件, dst为文件存储路径，size 为文件大小。上下文关闭
@@ -22,7 +28,7 @@ type Handler interface {
// 获取缩略图，可直接在ContentResponse中返回文件数据流，也可指
// 定为重定向
Thumb(ctx context.Context, path string) (*response.ContentResponse, error)
Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error)
// 获取外链/下载地址，
// url - 站点本身地址,

12
pkg/filesystem/driver/local/handler.go
View File

@@ -12,6 +12,7 @@ import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
@@ -194,15 +195,20 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
file, err := handler.Get(ctx, path+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
if file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusNotExist {
// Tell invoker to generate a thumb
return nil, driver.ErrorThumbNotExist
}
thumbFile, err := handler.Get(ctx, file.SourceName+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"))
if err != nil {
return nil, err
}
return &response.ContentResponse{
Redirect: false,
Content: file,
Content: thumbFile,
}, nil
}

4
pkg/filesystem/driver/onedrive/handler.go
View File

@@ -136,7 +136,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -145,7 +145,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
return nil, errors.New("failed to get thumbnail size")
}
res, err := handler.Client.GetThumbURL(ctx, path, thumbSize[0], thumbSize[1])
res, err := handler.Client.GetThumbURL(ctx, file.SourceName, thumbSize[0], thumbSize[1])
if err != nil {
// 如果出现异常，就清空文件的pic_info
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {

4
pkg/filesystem/driver/oss/handler.go
View File

@@ -293,7 +293,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
@@ -312,7 +312,7 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
thumbOption := []oss.Option{oss.Process(thumbParam)}
thumbURL, err := handler.signSourceURL(
ctx,
path,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
thumbOption,
)

6
pkg/filesystem/driver/qiniu/handler.go
View File

@@ -230,7 +230,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -239,12 +239,12 @@ func (handler *Driver) Thumb(ctx context.Context, path string) (*response.Conten
return nil, errors.New("无法获取缩略图尺寸设置")
}
path = fmt.Sprintf("%s?imageView2/1/w/%d/h/%d", path, thumbSize[0], thumbSize[1])
thumb := fmt.Sprintf("%s?imageView2/1/w/%d/h/%d", file.SourceName, thumbSize[0], thumbSize[1])
return &response.ContentResponse{
Redirect: true,
URL: handler.signSourceURL(
ctx,
path,
thumb,
int64(model.GetIntSetting("preview_timeout", 60)),
),
}, nil

4
pkg/filesystem/driver/remote/handler.go
View File

@@ -204,8 +204,8 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(file.SourceName))
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
ttl := model.GetIntSetting("preview_timeout", 60)
signedThumbURL, err := auth.SignURI(handler.AuthInstance, thumbURL, int64(ttl))

2
pkg/filesystem/driver/s3/handler.go
View File

@@ -264,7 +264,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
}

2
pkg/filesystem/driver/shadow/masterinslave/handler.go
View File

@@ -39,7 +39,7 @@ func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}

2
pkg/filesystem/driver/shadow/slaveinmaster/handler.go
View File

@@ -102,7 +102,7 @@ func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}

4
pkg/filesystem/driver/upyun/handler.go
View File

@@ -220,7 +220,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
@@ -232,7 +232,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
thumbParam := fmt.Sprintf("!/fwfh/%dx%d", thumbSize[0], thumbSize[1])
thumbURL, err := handler.Source(
ctx,
path+thumbParam,
file.SourceName+thumbParam,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,

16
pkg/filesystem/hooks.go
View File

@@ -184,7 +184,6 @@ func SlaveAfterUpload(session *serializer.UploadSession) Hook {
Name: fileInfo.FileName,
SourceName: fileInfo.SavePath,
}
fs.GenerateThumbnail(ctx, &file)
if session.Callback == "" {
return nil
@@ -231,21 +230,6 @@ func GenericAfterUpload(ctx context.Context, fs *FileSystem, fileHeader fsctx.Fi
return nil
}
// HookGenerateThumb 生成缩略图
func HookGenerateThumb(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
// 异步尝试生成缩略图
fileMode := fileHeader.Info().Model.(*model.File)
if fs.Policy.IsThumbGenerateNeeded() {
fs.recycleLock.Lock()
go func() {
defer fs.recycleLock.Unlock()
_, _ = fs.Handler.Delete(ctx, []string{fileMode.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")})
fs.GenerateThumbnail(ctx, fileMode)
}()
}
return nil
}
// HookClearFileHeaderSize 将FileHeader大小设定为0
func HookClearFileHeaderSize(ctx context.Context, fs *FileSystem, fileHeader fsctx.FileHeader) error {
fileHeader.SetSize(0)

96
pkg/filesystem/image.go
View File

@@ -2,13 +2,15 @@ package filesystem
import (
"context"
"fmt"
"errors"
"io"
"sync"
"runtime"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/thumb"
@@ -20,14 +22,11 @@ import (
================
*/
// HandledExtension 可以生成缩略图的文件扩展名
var HandledExtension = []string{"jpg", "jpeg", "png", "gif"}
// GetThumb 获取文件的缩略图
func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentResponse, error) {
// 根据 ID 查找文件
err := fs.resetFileIDIfNotExist(ctx, id)
if err != nil || fs.FileTarget[0].PicInfo == "" {
if err != nil {
return &response.ContentResponse{
Redirect: false,
}, ErrObjectNotExist
@@ -36,12 +35,11 @@ func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentR
w, h := fs.GenerateThumbnailSize(0, 0)
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, [2]uint{w, h})
ctx = context.WithValue(ctx, fsctx.FileModelCtx, fs.FileTarget[0])
res, err := fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
// 本地存储策略出错时重新生成缩略图
if err != nil && fs.Policy.Type == "local" {
res, err := fs.Handler.Thumb(ctx, &fs.FileTarget[0])
if errors.Is(err, driver.ErrorThumbNotExist) {
// Regenerate thumb if the thumb is not initialized yet
fs.GenerateThumbnail(ctx, &fs.FileTarget[0])
res, err = fs.Handler.Thumb(ctx, fs.FileTarget[0].SourceName)
res, err = fs.Handler.Thumb(ctx, &fs.FileTarget[0])
}
if err == nil && conf.SystemConfig.Mode == "master" {
@@ -84,14 +82,8 @@ func (pool *Pool) releaseWorker() {
<-pool.worker
}
// GenerateThumbnail 尝试为本地策略文件生成缩略图并获取图像原始大小
// TODO 失败时，如果之前还有图像信息，则清除
// GenerateThumbnail generates thumb for given file, upload the thumb file back with given suffix
func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// 判断是否可以生成缩略图
if !IsInExtensionList(HandledExtension, file.Name) {
return
}
// 新建上下文
newCtx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -105,36 +97,50 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
getThumbWorker().addWorker()
defer getThumbWorker().releaseWorker()
image, err := thumb.NewThumbFromFile(source, file.Name)
if err != nil {
util.Log().Warning("Cannot generate thumb because of failed to parse image %q: %s", file.SourceName, err)
r, w := io.Pipe()
defer w.Close()
errChan := make(chan error, 1)
go func() {
errChan <- fs.Handler.Put(newCtx, &fsctx.FileStream{
Mode: fsctx.Overwrite,
File: io.NopCloser(r),
Seeker: nil,
SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"),
})
}()
if err = thumb.Generators.Generate(source, w, file.Name, model.GetSettingByNames(
"thumb_width",
"thumb_height",
"thumb_builtin_enabled",
"thumb_vips_enabled",
"thumb_ffmpeg_enabled",
"thumb_vips_path",
"thumb_ffmpeg_path",
)); err != nil {
util.Log().Warning("Failed to generate thumb for %s: %s", file.Name, err)
if errors.Is(err, thumb.ErrNotAvailable) {
// Mark this file as no thumb available
_ = updateThumbStatus(file, model.ThumbStatusNotAvailable)
}
return
}
// 获取原始图像尺寸
w, h := image.GetSize()
w.Close()
if err = <-errChan; err != nil {
util.Log().Warning("Failed to save thumb for %s: %s", file.Name, err)
return
}
// 生成缩略图
image.GetThumb(fs.GenerateThumbnailSize(w, h))
// 保存到文件
err = image.Save(util.RelativePath(file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
image = nil
if model.IsTrueVal(model.GetSettingByName("thumb_gc_after_gen")) {
util.Log().Debug("GenerateThumbnail runtime.GC")
runtime.GC()
}
if err != nil {
util.Log().Warning("Failed to save thumb: %s", err)
return
}
// 更新文件的图像信息
if file.Model.ID > 0 {
err = file.UpdatePicInfo(fmt.Sprintf("%d,%d", w, h))
} else {
file.PicInfo = fmt.Sprintf("%d,%d", w, h)
}
// Mark this file as thumb available
err = updateThumbStatus(file, model.ThumbStatusExist)
// 失败时删除缩略图文件
if err != nil {
@@ -144,5 +150,17 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) {
// GenerateThumbnailSize 获取要生成的缩略图的尺寸
func (fs *FileSystem) GenerateThumbnailSize(w, h int) (uint, uint) {
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_width", 300))
return uint(model.GetIntSetting("thumb_width", 400)), uint(model.GetIntSetting("thumb_height", 300))
}
func updateThumbStatus(file *model.File, status string) error {
if file.Model.ID > 0 {
return file.UpdateMetadata(map[string]string{
model.ThumbStatusMetadataKey: status,
})
} else {
file.MetadataSerialized[model.ThumbStatusMetadataKey] = status
}
return nil
}

3
pkg/filesystem/manage.go
View File

@@ -341,7 +341,6 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
ID: hashid.HashID(subFolder.ID, hashid.FolderID),
Name: subFolder.Name,
Path: processedPath,
Pic: "",
Size: 0,
Type: "dir",
Date: subFolder.UpdatedAt,
@@ -363,7 +362,7 @@ func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []mo
ID: hashid.HashID(file.ID, hashid.FileID),
Name: file.Name,
Path: processedPath,
Pic: file.PicInfo,
Thumb: file.ShouldLoadThumb(),
Size: file.Size,
Type: "file",
Date: file.UpdatedAt,

1
pkg/filesystem/upload.go
View File

@@ -210,7 +210,6 @@ func (fs *FileSystem) UploadFromStream(ctx context.Context, file *fsctx.FileStre
fs.Use("BeforeUpload", HookValidateCapacity)
fs.Use("AfterUploadCanceled", HookDeleteTempFile)
fs.Use("AfterUpload", GenericAfterUpload)
fs.Use("AfterUpload", HookGenerateThumb)
fs.Use("AfterValidateFailed", HookDeleteTempFile)
}
fs.Lock.Unlock()