Refactor: factory method for OSS client

Fix: use HTTPS schema by default in OSS client Feat: new handler for Qiniu policy
2026-01-26 09:34:57 +08:00 · 2022-03-20 11:26:26 +08:00
parent 0df9529b32
commit 07f13cc350
13 changed files with 221 additions and 206 deletions
--- a/pkg/filesystem/driver/onedrive/handler.go
+++ b/pkg/filesystem/driver/onedrive/handler.go
@@ -234,7 +234,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 	// 监控回调及上传
 	go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)

-	uploadSession.OneDriveUploadURL = uploadURL
+	uploadSession.UploadURL = uploadURL
 	return &serializer.UploadCredential{
 		SessionID:  uploadSession.Key,
 		ChunkSize:  handler.Policy.OptionsSerialized.ChunkSize,
@@ -244,5 +244,5 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria

 // 取消上传凭证
 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
-	return handler.Client.DeleteUploadSession(ctx, uploadSession.OneDriveUploadURL)
+	return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
 }
--- a/pkg/filesystem/driver/oss/handler.go
+++ b/pkg/filesystem/driver/oss/handler.go
@@ -56,13 +56,17 @@ const (
 	VersionID key = iota
 )

-// CORS 创建跨域策略
-func (handler *Driver) CORS() error {
-	// 初始化客户端
-	if err := handler.InitOSSClient(false); err != nil {
-		return err
+func NewDriver(policy *model.Policy) (*Driver, error) {
+	driver := &Driver{
+		Policy:     policy,
+		HTTPClient: request.NewClient(),
 	}

+	return driver, driver.InitOSSClient(false)
+}
+
+// CORS 创建跨域策略
+func (handler *Driver) CORS() error {
 	return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
 		{
 			AllowedOrigin: []string{"*"},
@@ -86,39 +90,31 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
 		return errors.New("存储策略为空")
 	}

-	if handler.client == nil {
-		// 决定是否使用内网 Endpoint
-		endpoint := handler.Policy.Server
-		if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
-			endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
-		}
-
-		// 初始化客户端
-		client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
-		if err != nil {
-			return err
-		}
-		handler.client = client
-
-		// 初始化存储桶
-		bucket, err := client.Bucket(handler.Policy.BucketName)
-		if err != nil {
-			return err
-		}
-		handler.bucket = bucket
-
+	// 决定是否使用内网 Endpoint
+	endpoint := handler.Policy.Server
+	if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
+		endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
 	}

+	// 初始化客户端
+	client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
+	if err != nil {
+		return err
+	}
+	handler.client = client
+
+	// 初始化存储桶
+	bucket, err := client.Bucket(handler.Policy.BucketName)
+	if err != nil {
+		return err
+	}
+	handler.bucket = bucket
+
 	return nil
 }

 // List 列出OSS上的文件
-func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
-	// 初始化客户端
-	if err := handler.InitOSSClient(false); err != nil {
-		return nil, err
-	}
-
+func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
 	// 列取文件
 	base = strings.TrimPrefix(base, "/")
 	if base != "" {
@@ -185,7 +181,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
 }

 // Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
 	// 通过VersionID禁止缓存
 	ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())

@@ -228,15 +224,10 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
 }

 // Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
+func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
 	defer file.Close()
 	fileInfo := file.Info()

-	// 初始化客户端
-	if err := handler.InitOSSClient(false); err != nil {
-		return err
-	}
-
 	// 凭证有效期
 	credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)

@@ -247,12 +238,12 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
 		oss.ForbidOverWrite(!overwrite),
 	}

-	// 上传文件
-	//err := handler.bucket.PutObject(fileInfo.SavePath, file, options...)
-	//if err != nil {
-	//	return err
-	//}
+	// 小文件直接上传
+	if fileInfo.Size < MultiPartUploadThreshold {
+		return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
+	}

+	// 超过阈值时使用分片上传
 	imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
 	if err != nil {
 		return fmt.Errorf("failed to initiate multipart upload: %w", err)
@@ -280,12 +271,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {

 // Delete 删除一个或多个文件，
 // 返回未删除的文件
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
-	// 初始化客户端
-	if err := handler.InitOSSClient(false); err != nil {
-		return files, err
-	}
-
+func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
 	// 删除文件
 	delRes, err := handler.bucket.DeleteObjects(files)

@@ -303,7 +289,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
 }

 // Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
 	// 初始化客户端
 	if err := handler.InitOSSClient(true); err != nil {
 		return nil, err
@@ -337,7 +323,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
 }

 // Source 获取外链URL
-func (handler Driver) Source(
+func (handler *Driver) Source(
 	ctx context.Context,
 	path string,
 	baseURL url.URL,
@@ -382,7 +368,7 @@ func (handler Driver) Source(
 	return handler.signSourceURL(ctx, path, ttl, signOptions)
 }

-func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
+func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
 	signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
 	if err != nil {
 		return "", err
@@ -394,9 +380,6 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 		return "", err
 	}

-	// 优先使用https
-	finalURL.Scheme = "https"
-
 	// 公有空间替换掉Key及不支持的头
 	if !handler.Policy.IsPrivate {
 		query := finalURL.Query()
@@ -420,10 +403,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 }

 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
-	if err := handler.InitOSSClient(false); err != nil {
-		return nil, err
-	}
+func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {

 	// 生成回调地址
 	siteURL := model.GetSiteURL()
@@ -452,7 +432,7 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
 	}
-	uploadSession.OSSUploadID = imur.UploadID
+	uploadSession.UploadID = imur.UploadID

 	// 为每个分片签名上传 URL
 	chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{})
@@ -496,9 +476,6 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 }

 // 取消上传凭证
-func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
-	if err := handler.InitOSSClient(false); err != nil {
-		return err
-	}
-	return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.OSSUploadID, Key: uploadSession.SavePath}, nil)
+func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
+	return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
 }
--- a/pkg/filesystem/driver/qiniu/handler.go
+++ b/pkg/filesystem/driver/qiniu/handler.go
@@ -2,6 +2,7 @@ package qiniu

 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"net/http"
@@ -16,17 +17,31 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
 	"github.com/cloudreve/Cloudreve/v3/pkg/request"
 	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
-	"github.com/qiniu/api.v7/v7/auth/qbox"
-	"github.com/qiniu/api.v7/v7/storage"
+	"github.com/qiniu/go-sdk/v7/auth/qbox"
+	"github.com/qiniu/go-sdk/v7/storage"
 )

 // Driver 本地策略适配器
 type Driver struct {
 	Policy *model.Policy
+	mac    *qbox.Mac
+	cfg    *storage.Config
+	bucket *storage.BucketManager
+}
+
+func NewDriver(policy *model.Policy) *Driver {
+	mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
+	cfg := &storage.Config{UseHTTPS: true}
+	return &Driver{
+		Policy: policy,
+		mac:    mac,
+		cfg:    cfg,
+		bucket: storage.NewBucketManager(mac, cfg),
+	}
 }

 // List 列出给定路径下的文件
-func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
+func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
 	base = strings.TrimPrefix(base, "/")
 	if base != "" {
 		base += "/"
@@ -42,14 +57,8 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
 		delimiter = "/"
 	}

-	mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
-	cfg := storage.Config{
-		UseHTTPS: true,
-	}
-	bucketManager := storage.NewBucketManager(mac, &cfg)
-
 	for {
-		entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
+		entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
 			handler.Policy.BucketName,
 			base, delimiter, marker, 1000)
 		if err != nil {
@@ -99,7 +108,7 @@ func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]
 }

 // Get 获取文件
-func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
 	// 给文件名加上随机参数以强制拉取
 	path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())

@@ -143,7 +152,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
 }

 // Put 将文件流保存到指定目录
-func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
+func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
 	defer file.Close()

 	// 凭证有效期
@@ -151,9 +160,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {

 	// 生成上传策略
 	fileInfo := file.Info()
+	scope := handler.Policy.BucketName
+	if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
+		scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
+	}
+
 	putPolicy := storage.PutPolicy{
 		// 指定为覆盖策略
-		Scope:        fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath),
+		Scope:        scope,
 		SaveKey:      fileInfo.SavePath,
 		ForceSaveKey: true,
 		FsizeLimit:   int64(fileInfo.Size),
@@ -164,7 +178,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
 	}

 	// 生成上传凭证
-	token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
+	token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
 	if err != nil {
 		return err
 	}
@@ -178,7 +192,7 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
 	}

 	// 开始上传
-	err = formUploader.Put(ctx, &ret, token.Token, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
+	err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
 	if err != nil {
 		return err
 	}
@@ -188,19 +202,14 @@ func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {

 // Delete 删除一个或多个文件，
 // 返回未删除的文件
-func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
 	// TODO 大于一千个文件需要分批发送
 	deleteOps := make([]string, 0, len(files))
 	for _, key := range files {
 		deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
 	}

-	mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
-	cfg := storage.Config{
-		UseHTTPS: true,
-	}
-	bucketManager := storage.NewBucketManager(mac, &cfg)
-	rets, err := bucketManager.Batch(deleteOps)
+	rets, err := handler.bucket.Batch(deleteOps)

 	// 处理删除结果
 	if err != nil {
@@ -217,7 +226,7 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
 }

 // Thumb 获取文件缩略图
-func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+func (handler *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
 	var (
 		thumbSize = [2]uint{400, 300}
 		ok        = false
@@ -238,7 +247,7 @@ func (handler Driver) Thumb(ctx context.Context, path string) (*response.Content
 }

 // Source 获取外链URL
-func (handler Driver) Source(
+func (handler *Driver) Source(
 	ctx context.Context,
 	path string,
 	baseURL url.URL,
@@ -261,12 +270,11 @@ func (handler Driver) Source(
 	return handler.signSourceURL(ctx, path, ttl), nil
 }

-func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
+func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
 	var sourceURL string
 	if handler.Policy.IsPrivate {
-		mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
 		deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
-		sourceURL = storage.MakePrivateURL(mac, handler.Policy.BaseURL, path, deadline)
+		sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
 	} else {
 		sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
 	}
@@ -274,19 +282,20 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
 }

 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
 	apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
 	apiURL := siteURL.ResolveReference(apiBaseURI)

 	// 创建上传策略
+	fileInfo := file.Info()
 	putPolicy := storage.PutPolicy{
 		Scope:            handler.Policy.BucketName,
 		CallbackURL:      apiURL.String(),
-		CallbackBody:     `{"name":"$(fname)","source_name":"$(key)","size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
+		CallbackBody:     `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
 		CallbackBodyType: "application/json",
-		SaveKey:          file.Info().SavePath,
+		SaveKey:          fileInfo.SavePath,
 		ForceSaveKey:     true,
 		FsizeLimit:       int64(handler.Policy.MaxSize),
 	}
@@ -295,21 +304,46 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 		putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
 	}

-	return handler.getUploadCredential(ctx, putPolicy, ttl)
+	credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init parts: %w", err)
+	}
+
+	credential.SessionID = uploadSession.Key
+	credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
+
+	uploadSession.UploadURL = credential.UploadURLs[0]
+	uploadSession.Credential = credential.Credential
+
+	return credential, nil
 }

-// getUploadCredential 签名上传策略
-func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (*serializer.UploadCredential, error) {
+// getUploadCredential 签名上传策略并创建上传会话
+func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
+	// 上传凭证
 	policy.Expires = uint64(TTL)
-	mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
-	upToken := policy.UploadToken(mac)
+	upToken := policy.UploadToken(handler.mac)
+
+	// 初始化分片上传
+	resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
+	upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &storage.InitPartsRet{}
+	if resume {
+		err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
+	}

 	return &serializer.UploadCredential{
-		Token: upToken,
-	}, nil
+		UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
+		Credential: upToken,
+	}, err
 }

 // 取消上传凭证
 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
-	return nil
+	resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
+	return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
 }
--- a/pkg/filesystem/filesystem.go
+++ b/pkg/filesystem/filesystem.go
@@ -144,16 +144,12 @@ func (fs *FileSystem) DispatchHandler() error {

 		fs.Handler = handler
 	case "qiniu":
-		fs.Handler = qiniu.Driver{
-			Policy: currentPolicy,
-		}
+		fs.Handler = qiniu.NewDriver(currentPolicy)
 		return nil
 	case "oss":
-		fs.Handler = oss.Driver{
-			Policy:     currentPolicy,
-			HTTPClient: request.NewClient(),
-		}
-		return nil
+		handler, err := oss.NewDriver(currentPolicy)
+		fs.Handler = handler
+		return err
 	case "upyun":
 		fs.Handler = upyun.Driver{
 			Policy: currentPolicy,
--- a/pkg/filesystem/upload.go
+++ b/pkg/filesystem/upload.go
@@ -91,40 +91,17 @@ func (fs *FileSystem) Upload(ctx context.Context, file *fsctx.FileStream) (err e
 // TODO 完善测试
 func (fs *FileSystem) GenerateSavePath(ctx context.Context, file fsctx.FileHeader) string {
 	fileInfo := file.Info()
-
-	if fs.User.Model.ID != 0 {
-		return path.Join(
-			fs.Policy.GeneratePath(
-				fs.User.Model.ID,
-				fileInfo.VirtualPath,
-			),
-			fs.Policy.GenerateFileName(
-				fs.User.Model.ID,
-				fileInfo.FileName,
-			),
-		)
-	}
-
-	// 匿名文件系统尝试根据上下文中的上传策略生成路径
-	var anonymousPolicy model.Policy
-	if policy, ok := ctx.Value(fsctx.UploadPolicyCtx).(serializer.UploadPolicy); ok {
-		anonymousPolicy = model.Policy{
-			Type:         "remote",
-			AutoRename:   policy.AutoRename,
-			DirNameRule:  policy.SavePath,
-			FileNameRule: policy.FileName,
-		}
-	}
 	return path.Join(
-		anonymousPolicy.GeneratePath(
-			0,
-			"",
+		fs.Policy.GeneratePath(
+			fs.User.Model.ID,
+			fileInfo.VirtualPath,
 		),
-		anonymousPolicy.GenerateFileName(
-			0,
+		fs.Policy.GenerateFileName(
+			fs.User.Model.ID,
 			fileInfo.FileName,
 		),
 	)
+
 }

 // CancelUpload 监测客户端取消上传