Feat: upyun download / thumb / sign

This commit is contained in:
HFO4
2020-01-18 14:08:43 +08:00
parent 84a6218d3a
commit fa3b51096a
30 changed files with 212 additions and 146 deletions

View File

@@ -0,0 +1,279 @@
package remote
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"io"
"net/http"
"net/url"
"path"
"strings"
)
// Driver 远程存储策略适配器
type Driver struct {
Client request.Client
Policy *model.Policy
AuthInstance auth.Auth
}
// getAPIUrl 获取接口请求地址
func (handler Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return ""
}
var controller *url.URL
switch scope {
case "delete":
controller, _ = url.Parse("/api/v3/slave/delete")
case "thumb":
controller, _ = url.Parse("/api/v3/slave/thumb")
default:
controller = serverURL
}
for _, r := range routes {
controller.Path = path.Join(controller.Path, r)
}
return serverURL.ResolveReference(controller).String()
}
// Get 获取文件内容
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 尝试获取速度限制 TODO 是否需要在这里限制?
speedLimit := 0
if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
speedLimit = user.Group.SpeedLimit
}
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, url.URL{}, 0, true, speedLimit)
if err != nil {
return nil, err
}
// 获取文件数据流
resp, err := handler.Client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略
policy := serializer.UploadPolicy{
SavePath: path.Dir(dst),
FileName: path.Base(dst),
AutoRename: false,
MaxSize: size,
}
credential, err := handler.getUploadCredential(ctx, policy, int64(credentialTTL))
if err != nil {
return err
}
// 对文件名进行URLEncode
fileName, err := url.QueryUnescape(path.Base(dst))
if err != nil {
return err
}
// 上传文件
resp, err := handler.Client.Request(
"POST",
handler.Policy.GetUploadURL(),
file,
request.WithHeader(map[string][]string{
"Authorization": {credential.Token},
"X-Policy": {credential.Policy},
"X-FileName": {fileName},
}),
request.WithContentLength(int64(size)),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return errors.New(resp.Msg)
}
return nil
}
// Delete 删除一个或多个文件,
// 返回未删除的文件,及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 封装接口请求正文
reqBody := serializer.RemoteDeleteRequest{
Files: files,
}
reqBodyEncoded, err := json.Marshal(reqBody)
if err != nil {
return files, err
}
// 发送删除请求
bodyReader := strings.NewReader(string(reqBodyEncoded))
signTTL := model.GetIntSetting("slave_api_timeout", 60)
resp, err := handler.Client.Request(
"POST",
handler.getAPIUrl("delete"),
bodyReader,
request.WithCredential(handler.AuthInstance, int64(signTTL)),
).CheckHTTPResponse(200).GetResponse()
if err != nil {
return files, err
}
// 处理删除结果
var reqResp serializer.Response
err = json.Unmarshal([]byte(resp), &reqResp)
if err != nil {
return files, err
}
if reqResp.Code != 0 {
var failedResp serializer.RemoteDeleteRequest
if failed, ok := reqResp.Data.(string); ok {
err = json.Unmarshal([]byte(failed), &failedResp)
if err == nil {
return failedResp.Files, errors.New(reqResp.Error)
}
}
return files, errors.New("未知的返回结果格式")
}
return []string{}, nil
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
thumbURL := handler.getAPIUrl("thumb") + "/" + sourcePath
ttl := model.GetIntSetting("preview_timeout", 60)
signedThumbURL, err := auth.SignURI(handler.AuthInstance, thumbURL, int64(ttl))
if err != nil {
return nil, err
}
return &response.ContentResponse{
Redirect: true,
URL: signedThumbURL.String(),
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
baseURL url.URL,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
// 尝试从上下文获取文件名
fileName := "file"
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return "", errors.New("无法解析远程服务端地址")
}
var (
signedURI *url.URL
controller = "/api/v3/slave/download"
)
if !isDownload {
controller = "/api/v3/slave/source"
}
// 签名下载地址
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
signedURI, err = auth.SignURI(
handler.AuthInstance,
fmt.Sprintf("%s/%d/%s/%s", controller, speed, sourcePath, fileName),
ttl,
)
if err != nil {
return "", serializer.NewError(serializer.CodeEncryptError, "无法对URL进行签名", err)
}
finalURL := serverURL.ResolveReference(signedURI).String()
return finalURL, nil
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 生成上传策略
policy := serializer.UploadPolicy{
SavePath: handler.Policy.DirNameRule,
FileName: handler.Policy.FileNameRule,
AutoRename: handler.Policy.AutoRename,
MaxSize: handler.Policy.MaxSize,
AllowedExtension: handler.Policy.OptionsSerialized.FileType,
CallbackURL: apiURL.String(),
}
return handler.getUploadCredential(ctx, policy, TTL)
}
func (handler Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) {
policyEncoded, err := policy.EncodeUploadPolicy()
if err != nil {
return serializer.UploadCredential{}, err
}
// 签名上传策略
uploadRequest, _ := http.NewRequest("POST", "/api/v3/slave/upload", nil)
uploadRequest.Header = map[string][]string{
"X-Policy": {policyEncoded},
}
auth.SignRequest(handler.AuthInstance, uploadRequest, TTL)
if credential, ok := uploadRequest.Header["Authorization"]; ok && len(credential) == 1 {
return serializer.UploadCredential{
Token: credential[0],
Policy: policyEncoded,
}, nil
}
return serializer.UploadCredential{}, errors.New("无法签名上传策略")
}

View File

@@ -0,0 +1,354 @@
package remote
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
MaxSize: 10,
AutoRename: true,
DirNameRule: "dir",
FileNameRule: "file",
OptionsSerialized: model.PolicyOption{
FileType: []string{"txt"},
},
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 成功
{
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
credential, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
policy, err := serializer.DecodeUploadPolicy(credential.Policy)
asserts.NoError(err)
asserts.Equal("http://test.cloudreve.org/api/v3/callback/remote/123", policy.CallbackURL)
asserts.Equal(uint64(10), policy.MaxSize)
asserts.Equal(true, policy.AutoRename)
asserts.Equal("dir", policy.SavePath)
asserts.Equal("file", policy.FileName)
asserts.Equal([]string{"txt"}, policy.AllowedExtension)
}
}
func TestHandler_Source(t *testing.T) {
asserts := assert.New(t)
auth.General = auth.HMACAuth{SecretKey: []byte("test")}
// 无法获取上下文
{
handler := Driver{
Policy: &model.Policy{Server: "/"},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
res, err := handler.Source(ctx, "", url.URL{}, 0, true, 0)
asserts.NoError(err)
asserts.NotEmpty(res)
}
// 成功
{
handler := Driver{
Policy: &model.Policy{Server: "/"},
AuthInstance: auth.HMACAuth{},
}
file := model.File{
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, true, 0)
asserts.NoError(err)
asserts.Contains(res, "api/v3/slave/download/0")
}
// 成功 预览
{
handler := Driver{
Policy: &model.Policy{Server: "/"},
AuthInstance: auth.HMACAuth{},
}
file := model.File{
SourceName: "1.txt",
}
ctx := context.WithValue(context.Background(), fsctx.FileModelCtx, file)
res, err := handler.Source(ctx, "", url.URL{}, 10, false, 0)
asserts.NoError(err)
asserts.Contains(res, "api/v3/slave/source/0")
}
}
type ClientMock struct {
testMock.Mock
}
func (m ClientMock) Request(method, target string, body io.Reader, opts ...request.Option) *request.Response {
args := m.Called(method, target, body, opts)
return args.Get(0).(*request.Response)
}
func TestHandler_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
cache.Set("setting_slave_api_timeout", "60", 0)
// 成功
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/delete",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
failed, err := handler.Delete(ctx, []string{"/test1.txt", "test2.txt"})
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Len(failed, 0)
}
// 结果解析失败
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/delete",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":203}`)),
},
})
handler.Client = clientMock
failed, err := handler.Delete(ctx, []string{"/test1.txt", "test2.txt"})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Len(failed, 2)
}
// 一个失败
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/delete",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":203,"data":"{\"files\":[\"1\"]}"}`)),
},
})
handler.Client = clientMock
failed, err := handler.Delete(ctx, []string{"/test1.txt", "test2.txt"})
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Len(failed, 1)
}
}
func TestHandler_Get(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
// 成功
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
nil,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
resp, err := handler.Get(ctx, "/test.txt")
clientMock.AssertExpectations(t)
asserts.NotNil(resp)
asserts.NoError(err)
}
// 请求失败
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
nil,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
resp, err := handler.Get(ctx, "/test.txt")
clientMock.AssertExpectations(t)
asserts.Nil(resp)
asserts.Error(err)
}
}
func TestHandler_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_upload_credential_timeout", "3600", 0))
// 成功
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.NoError(err)
}
// 请求失败
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
// 返回错误
{
ctx = context.WithValue(ctx, fsctx.UserCtx, model.User{})
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/upload",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":1}`)),
},
})
handler.Client = clientMock
err := handler.Put(ctx, ioutil.NopCloser(strings.NewReader("test input file")), "/", 15)
clientMock.AssertExpectations(t)
asserts.Error(err)
}
}
func TestHandler_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
Type: "remote",
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
asserts.NoError(cache.Set("setting_preview_timeout", "60", 0))
resp, err := handler.Thumb(ctx, "/1.txt")
asserts.NoError(err)
asserts.True(resp.Redirect)
}