Feat: aria2 download and transfer in slave node (#1040)

* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings

pkg/filesystem/driver/handler.go

@@ -0,0 +1,39 @@
+package driver
+
+import (
+	"context"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
+	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+	"io"
+	"net/url"
+)
+
+// Handler 存储策略适配器
+type Handler interface {
+	// 上传文件, dst为文件存储路径，size 为文件大小。上下文关闭
+	// 时，应取消上传并清理临时文件
+	Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error
+
+	// 删除一个或多个给定路径的文件，返回删除失败的文件路径列表及错误
+	Delete(ctx context.Context, files []string) ([]string, error)
+
+	// 获取文件内容
+	Get(ctx context.Context, path string) (response.RSCloser, error)
+
+	// 获取缩略图，可直接在ContentResponse中返回文件数据流，也可指
+	// 定为重定向
+	Thumb(ctx context.Context, path string) (*response.ContentResponse, error)
+
+	// 获取外链/下载地址，
+	// url - 站点本身地址,
+	// isDownload - 是否直接下载
+	Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
+
+	// Token 获取有效期为ttl的上传凭证和签名，同时回调会话有效期为sessionTTL
+	Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
+
+	// List 递归列取远程端path路径下文件、目录，不包含path本身，
+	// 返回的对象路径以path作为起始根目录.
+	// recursive - 是否递归列出
+	List(ctx context.Context, path string, recursive bool) ([]response.Object, error)
+}

pkg/filesystem/driver/onedrive/client.go

@@ -55,7 +55,7 @@ func NewClient(policy *model.Policy) (*Client, error) {
 		ClientID:     policy.BucketName,
 		ClientSecret: policy.SecretKey,
 		Redirect:     policy.OptionsSerialized.OdRedirect,
-		Request:      request.HTTPClient{},
+		Request:      request.NewClient(),
 	}
 
 	if client.Endpoints.DriverResource == "" {

pkg/filesystem/driver/onedrive/handler.go

@@ -14,6 +14,7 @@ import (
 	model "github.com/cloudreve/Cloudreve/v3/models"
 	"github.com/cloudreve/Cloudreve/v3/pkg/auth"
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
 	"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -27,6 +28,16 @@ type Driver struct {
 	HTTPClient request.Client
 }
 
+// NewDriver 从存储策略初始化新的Driver实例
+func NewDriver(policy *model.Policy) (driver.Handler, error) {
+	client, err := NewClient(policy)
+	return Driver{
+		Policy:     policy,
+		Client:     client,
+		HTTPClient: request.NewClient(),
+	}, err
+}
+
 // List 列取项目
 func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
 	base = strings.TrimPrefix(base, "/")

pkg/filesystem/driver/onedrive/lock.go

@@ -0,0 +1,25 @@
+package onedrive
+
+import "sync"
+
+// CredentialLock 针对存储策略凭证的锁
+type CredentialLock interface {
+	Lock(uint)
+	Unlock(uint)
+}
+
+var GlobalMutex = mutexMap{}
+
+type mutexMap struct {
+	locks sync.Map
+}
+
+func (m *mutexMap) Lock(id uint) {
+	lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
+	lock.(*sync.Mutex).Lock()
+}
+
+func (m *mutexMap) Unlock(id uint) {
+	lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
+	lock.(*sync.Mutex).Unlock()
+}

pkg/filesystem/driver/onedrive/oauth.go

@@ -10,7 +10,9 @@ import (
 	"time"
 
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/request"
+	"github.com/cloudreve/Cloudreve/v3/pkg/slave"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 )
 
@@ -124,6 +126,13 @@ func (client *Client) ObtainToken(ctx context.Context, opts ...Option) (*Credent
 
 // UpdateCredential 更新凭证，并检查有效期
 func (client *Client) UpdateCredential(ctx context.Context) error {
+	if conf.SystemConfig.Mode == "slave" {
+		return client.fetchCredentialFromMaster(ctx)
+	}
+
+	GlobalMutex.Lock(client.Policy.ID)
+	defer GlobalMutex.Unlock(client.Policy.ID)
+
 	// 如果已存在凭证
 	if client.Credential != nil && client.Credential.AccessToken != "" {
 		// 检查已有凭证是否过期
@@ -160,11 +169,21 @@ func (client *Client) UpdateCredential(ctx context.Context) error {
 	client.Credential = credential
 
 	// 更新存储策略的 RefreshToken
-	client.Policy.AccessKey = credential.RefreshToken
-	client.Policy.SaveAndClearCache()
+	client.Policy.UpdateAccessKeyAndClearCache(credential.RefreshToken)
 
 	// 更新缓存
 	cache.Set("onedrive_"+client.ClientID, *credential, int(expires))
 
 	return nil
 }
+
+// UpdateCredential 更新凭证，并检查有效期
+func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
+	res, err := slave.DefaultController.GetOneDriveToken(client.Policy.MasterID, client.Policy.ID)
+	if err != nil {
+		return err
+	}
+
+	client.Credential = &Credential{AccessToken: res}
+	return nil
+}

pkg/filesystem/driver/oss/callback.go

@@ -42,7 +42,7 @@ func GetPublicKey(r *http.Request) ([]byte, error) {
 	}
 
 	// 获取公钥
-	client := request.HTTPClient{}
+	client := request.NewClient()
 	body, err := client.Request("GET", string(pubURL), nil).
 		CheckHTTPResponse(200).
 		GetResponse()

pkg/filesystem/driver/oss/handler_test.go

@@ -292,7 +292,7 @@ func TestDriver_Get(t *testing.T) {
 			BucketName: "test",
 			Server:     "oss-cn-shanghai.aliyuncs.com",
 		},
-		HTTPClient: request.HTTPClient{},
+		HTTPClient: request.NewClient(),
 	}
 	cache.Set("setting_preview_timeout", "3600", 0)
 

pkg/filesystem/driver/remote/handler.go

@@ -49,6 +49,7 @@ func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]
 		handler.getAPIUrl("list"),
 		bodyReader,
 		request.WithCredential(handler.AuthInstance, int64(signTTL)),
+		request.WithMasterMeta(),
 	).CheckHTTPResponse(200).DecodeResponse()
 	if err != nil {
 		return res, err
@@ -97,7 +98,7 @@ func (handler Driver) getAPIUrl(scope string, routes ...string) string {
 
 // Get 获取文件内容
 func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
-	// 尝试获取速度限制 TODO 是否需要在这里限制？
+	// 尝试获取速度限制
 	speedLimit := 0
 	if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
 		speedLimit = user.Group.SpeedLimit
@@ -116,6 +117,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
 		nil,
 		request.WithContext(ctx),
 		request.WithTimeout(time.Duration(0)),
+		request.WithMasterMeta(),
 	).CheckHTTPResponse(200).GetRSCloser()
 	if err != nil {
 		return nil, err
@@ -168,13 +170,15 @@ func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, s
 		handler.Policy.GetUploadURL(),
 		file,
 		request.WithHeader(map[string][]string{
-			"Authorization": {credential.Token},
-			"X-Policy":      {credential.Policy},
-			"X-FileName":    {fileName},
-			"X-Overwrite":   {overwrite},
+			"X-Policy":    {credential.Policy},
+			"X-FileName":  {fileName},
+			"X-Overwrite": {overwrite},
 		}),
 		request.WithContentLength(int64(size)),
 		request.WithTimeout(time.Duration(0)),
+		request.WithMasterMeta(),
+		request.WithSlaveMeta(handler.Policy.AccessKey),
+		request.WithCredential(handler.AuthInstance, int64(credentialTTL)),
 	).CheckHTTPResponse(200).DecodeResponse()
 	if err != nil {
 		return err
@@ -206,6 +210,8 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
 		handler.getAPIUrl("delete"),
 		bodyReader,
 		request.WithCredential(handler.AuthInstance, int64(signTTL)),
+		request.WithMasterMeta(),
+		request.WithSlaveMeta(handler.Policy.AccessKey),
 	).CheckHTTPResponse(200).GetResponse()
 	if err != nil {
 		return files, err

pkg/filesystem/driver/s3/handler.go

@@ -172,7 +172,7 @@ func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser,
 	}
 
 	// 获取文件数据流
-	client := request.HTTPClient{}
+	client := request.NewClient()
 	resp, err := client.Request(
 		"GET",
 		downloadURL,

pkg/filesystem/driver/shadow/masterinslave/errors.go

@@ -0,0 +1,7 @@
+package masterinslave
+
+import "errors"
+
+var (
+	ErrNotImplemented = errors.New("this method of shadowed policy is not implemented")
+)

pkg/filesystem/driver/shadow/masterinslave/handler.go

@@ -0,0 +1,56 @@
+package masterinslave
+
+import (
+	"context"
+	model "github.com/cloudreve/Cloudreve/v3/models"
+	"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
+	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+	"io"
+	"net/url"
+)
+
+// Driver 影子存储策略，用于在从机端上传文件
+type Driver struct {
+	master  cluster.Node
+	handler driver.Handler
+	policy  *model.Policy
+}
+
+// NewDriver 返回新的处理器
+func NewDriver(master cluster.Node, handler driver.Handler, policy *model.Policy) driver.Handler {
+	return &Driver{
+		master:  master,
+		handler: handler,
+		policy:  policy,
+	}
+}
+
+func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
+	return d.handler.Put(ctx, file, dst, size)
+}
+
+func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+	return d.handler.Delete(ctx, files)
+}
+
+func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+	return nil, ErrNotImplemented
+}
+
+func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+	return nil, ErrNotImplemented
+}
+
+func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error) {
+	return "", ErrNotImplemented
+}
+
+func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
+	return serializer.UploadCredential{}, ErrNotImplemented
+}
+
+func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
+	return nil, ErrNotImplemented
+}

pkg/filesystem/driver/shadow/slaveinmaster/errors.go

@@ -0,0 +1,9 @@
+package slaveinmaster
+
+import "errors"
+
+var (
+	ErrNotImplemented       = errors.New("this method of shadowed policy is not implemented")
+	ErrSlaveSrcPathNotExist = errors.New("cannot determine source file path in slave node")
+	ErrWaitResultTimeout    = errors.New("timeout waiting for slave transfer result")
+)

pkg/filesystem/driver/shadow/slaveinmaster/handler.go

@@ -0,0 +1,121 @@
+package slaveinmaster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	model "github.com/cloudreve/Cloudreve/v3/models"
+	"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
+	"github.com/cloudreve/Cloudreve/v3/pkg/mq"
+	"github.com/cloudreve/Cloudreve/v3/pkg/request"
+	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+	"io"
+	"net/url"
+	"time"
+)
+
+// Driver 影子存储策略，将上传任务指派给从机节点处理，并等待从机通知上传结果
+type Driver struct {
+	node    cluster.Node
+	handler driver.Handler
+	policy  *model.Policy
+	client  request.Client
+}
+
+// NewDriver 返回新的从机指派处理器
+func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy) driver.Handler {
+	var endpoint *url.URL
+	if serverURL, err := url.Parse(node.DBModel().Server); err == nil {
+		var controller *url.URL
+		controller, _ = url.Parse("/api/v3/slave")
+		endpoint = serverURL.ResolveReference(controller)
+	}
+
+	signTTL := model.GetIntSetting("slave_api_timeout", 60)
+	return &Driver{
+		node:    node,
+		handler: handler,
+		policy:  policy,
+		client: request.NewClient(
+			request.WithMasterMeta(),
+			request.WithTimeout(time.Duration(signTTL)*time.Second),
+			request.WithCredential(node.SlaveAuthInstance(), int64(signTTL)),
+			request.WithEndpoint(endpoint.String()),
+		),
+	}
+}
+
+// Put 将ctx中指定的从机物理文件由从机上传到目标存储策略
+func (d *Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
+	src, ok := ctx.Value(fsctx.SlaveSrcPath).(string)
+	if !ok {
+		return ErrSlaveSrcPathNotExist
+	}
+
+	req := serializer.SlaveTransferReq{
+		Src:    src,
+		Dst:    dst,
+		Policy: d.policy,
+	}
+
+	body, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	// 订阅转存结果
+	resChan := mq.GlobalMQ.Subscribe(req.Hash(model.GetSettingByName("siteID")), 0)
+	defer mq.GlobalMQ.Unsubscribe(req.Hash(model.GetSettingByName("siteID")), resChan)
+
+	res, err := d.client.Request("PUT", "task/transfer", bytes.NewReader(body)).
+		CheckHTTPResponse(200).
+		DecodeResponse()
+	if err != nil {
+		return err
+	}
+
+	if res.Code != 0 {
+		return serializer.NewErrorFromResponse(res)
+	}
+
+	// 等待转存结果或者超时
+	waitTimeout := model.GetIntSetting("slave_transfer_timeout", 172800)
+	select {
+	case <-time.After(time.Duration(waitTimeout) * time.Second):
+		return ErrWaitResultTimeout
+	case msg := <-resChan:
+		if msg.Event != serializer.SlaveTransferSuccess {
+			return errors.New(msg.Content.(serializer.SlaveTransferResult).Error)
+		}
+	}
+
+	return nil
+}
+
+func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
+	return d.handler.Delete(ctx, files)
+}
+
+func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
+	return nil, ErrNotImplemented
+}
+
+func (d *Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
+	return nil, ErrNotImplemented
+}
+
+func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error) {
+	return "", ErrNotImplemented
+}
+
+func (d *Driver) Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error) {
+	return serializer.UploadCredential{}, ErrNotImplemented
+}
+
+func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
+	return nil, ErrNotImplemented
+}