Feat: upload session recycle crontab job / API for cleanup all upload session

pkg/crontab/collect.go

@@ -1,6 +1,7 @@
 package crontab
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"strings"
@@ -8,6 +9,7 @@ import (
 
 	model "github.com/cloudreve/Cloudreve/v3/models"
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 )
 
@@ -53,3 +55,45 @@ func collectCache(store *cache.MemoStore) {
 	util.Log().Debug("清理内存缓存")
 	store.GarbageCollect()
 }
+
+func uploadSessionCollect() {
+	placeholders := model.GetUploadPlaceholderFiles(0)
+
+	// 将过期的上传会话按照用户分组
+	userToFiles := make(map[uint][]uint)
+	for _, file := range placeholders {
+		_, sessionExist := cache.Get(filesystem.UploadSessionCachePrefix + *file.UploadSessionID)
+		if sessionExist {
+			continue
+		}
+
+		if _, ok := userToFiles[file.UserID]; !ok {
+			userToFiles[file.UserID] = make([]uint, 0)
+		}
+
+		userToFiles[file.UserID] = append(userToFiles[file.UserID], file.ID)
+	}
+
+	// 删除过期的会话
+	for uid, filesIDs := range userToFiles {
+		user, err := model.GetUserByID(uid)
+		if err != nil {
+			util.Log().Warning("上传会话所属用户不存在, %s", err)
+			continue
+		}
+
+		fs, err := filesystem.NewFileSystem(&user)
+		if err != nil {
+			util.Log().Warning("无法初始化文件系统, %s", err)
+			continue
+		}
+
+		if err = fs.Delete(context.Background(), []uint{}, filesIDs, false); err != nil {
+			util.Log().Warning("无法删除上传会话, %s", err)
+		}
+
+		fs.Recycle()
+	}
+
+	util.Log().Info("定时任务 [cron_recycle_upload_session] 执行完毕")
+}

pkg/crontab/init.go

@@ -21,13 +21,18 @@ func Reload() {
 func Init() {
 	util.Log().Info("初始化定时任务...")
 	// 读取cron日程设置
-	options := model.GetSettingByNames("cron_garbage_collect")
+	options := model.GetSettingByNames(
+		"cron_garbage_collect",
+		"cron_recycle_upload_session",
+	)
 	Cron := cron.New()
 	for k, v := range options {
 		var handler func()
 		switch k {
 		case "cron_garbage_collect":
 			handler = garbageCollect
+		case "cron_recycle_upload_session":
+			handler = uploadSessionCollect
 		default:
 			util.Log().Warning("未知定时任务类型 [%s]，跳过", k)
 			continue

pkg/filesystem/driver/cos/handler.go

@@ -365,6 +365,11 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 
 }
 
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}
+
 // Meta 获取文件信息
 func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
 	res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})

pkg/filesystem/driver/handler.go

@@ -32,6 +32,9 @@ type Handler interface {
 	// Token 获取有效期为ttl的上传凭证和签名
 	Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error)
 
+	// CancelToken 取消已经创建的有状态上传凭证
+	CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error
+
 	// List 递归列取远程端path路径下文件、目录，不包含path本身，
 	// 返回的对象路径以path作为起始根目录.
 	// recursive - 是否递归列出

pkg/filesystem/driver/local/handler.go

@@ -260,3 +260,8 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 		ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
 	}, nil
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/onedrive/handler.go

@@ -249,3 +249,8 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 		Token:  apiURL.String(),
 	}, nil
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/oss/handler.go

@@ -463,3 +463,8 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
 		Token:     signature,
 	}, nil
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/qiniu/handler.go

@@ -308,3 +308,8 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy storage.Pu
 		Token: upToken,
 	}, nil
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/remote/handler.go

@@ -345,3 +345,8 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy serializer
 	}
 	return serializer.UploadCredential{}, errors.New("无法签名上传策略")
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/s3/handler.go

@@ -446,3 +446,8 @@ func (handler Driver) CORS() error {
 
 	return err
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/shadow/masterinslave/handler.go

@@ -54,3 +54,8 @@ func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer
 func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
 	return nil, ErrNotImplemented
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/shadow/slaveinmaster/handler.go

@@ -120,3 +120,8 @@ func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer
 func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
 	return nil, ErrNotImplemented
 }
+
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}

pkg/filesystem/driver/upyun/handler.go

@@ -335,6 +335,11 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 	return handler.getUploadCredential(ctx, putPolicy)
 }
 
+// 取消上传凭证
+func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return nil
+}
+
 func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy) (serializer.UploadCredential, error) {
 	// 生成上传策略
 	policyJSON, err := json.Marshal(policy)

pkg/filesystem/file.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	model "github.com/cloudreve/Cloudreve/v3/models"
+	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
 	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
@@ -177,23 +178,45 @@ func (fs *FileSystem) deleteGroupedFile(ctx context.Context, files map[uint][]*m
 
 	for policyID, toBeDeletedFiles := range files {
 		// 列举出需要物理删除的文件的物理路径
-		sourceNames := make([]string, 0, len(toBeDeletedFiles))
+		sourceNamesAll := make([]string, 0, len(toBeDeletedFiles))
+		sourceNamesDeleted := make([]string, 0, len(toBeDeletedFiles))
+		sourceNamesTryDeleted := make([]string, 0, len(toBeDeletedFiles))
+
 		for i := 0; i < len(toBeDeletedFiles); i++ {
-			sourceNames = append(sourceNames, toBeDeletedFiles[i].SourceName)
+			sourceNamesAll = append(sourceNamesAll, toBeDeletedFiles[i].SourceName)
+			if !(toBeDeletedFiles[i].UploadSessionID != nil && toBeDeletedFiles[i].Size == 0) {
+				sourceNamesDeleted = append(sourceNamesDeleted, toBeDeletedFiles[i].SourceName)
+			} else {
+				sourceNamesTryDeleted = append(sourceNamesTryDeleted, toBeDeletedFiles[i].SourceName)
+			}
+
+			if toBeDeletedFiles[i].UploadSessionID != nil {
+				if session, ok := cache.Get(UploadSessionCachePrefix + *toBeDeletedFiles[i].UploadSessionID); ok {
+					uploadSession := session.(serializer.UploadSession)
+					if err := fs.Handler.CancelToken(ctx, &uploadSession); err != nil {
+						util.Log().Warning("无法取消 [%s] 的上传会话: %s", err)
+					}
+
+					cache.Deletes([]string{*toBeDeletedFiles[i].UploadSessionID}, UploadSessionCachePrefix)
+				}
+
+			}
 		}
 
 		// 切换上传策略
 		fs.Policy = toBeDeletedFiles[0].GetPolicy()
 		err := fs.DispatchHandler()
 		if err != nil {
-			failed[policyID] = sourceNames
+			failed[policyID] = sourceNamesAll
 			continue
 		}
 
 		// 执行删除
-		failedFile, _ := fs.Handler.Delete(ctx, sourceNames)
+		failedFile, _ := fs.Handler.Delete(ctx, sourceNamesDeleted)
 		failed[policyID] = failedFile
 
+		// 尝试删除上传会话中大小为0的占位文件。如果失败也忽略
+		fs.Handler.Delete(ctx, sourceNamesTryDeleted)
 	}
 
 	return failed
@@ -208,7 +231,7 @@ func (fs *FileSystem) GroupFileByPolicy(ctx context.Context, files []model.File)
 			// 如果已存在分组，直接追加
 			policyGroup[files[key].PolicyID] = append(file, &files[key])
 		} else {
-			// 分布不存在，创建
+			// 分组不存在，创建
 			policyGroup[files[key].PolicyID] = make([]*model.File, 0)
 			policyGroup[files[key].PolicyID] = append(policyGroup[files[key].PolicyID], &files[key])
 		}

pkg/filesystem/manage.go

@@ -185,6 +185,7 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool
 	}
 
 	// 删除文件记录对应的分享记录
+	// TODO 先取消分享再删除文件
 	model.DeleteShareBySourceIDs(deletedFileIDs, false)
 
 	// 归还容量