Feat: aria2 download and transfer in slave node (#1040)

* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings

pkg/filesystem/driver/onedrive/client.go

@@ -55,7 +55,7 @@ func NewClient(policy *model.Policy) (*Client, error) {
 		ClientID:     policy.BucketName,
 		ClientSecret: policy.SecretKey,
 		Redirect:     policy.OptionsSerialized.OdRedirect,
-		Request:      request.HTTPClient{},
+		Request:      request.NewClient(),
 	}
 
 	if client.Endpoints.DriverResource == "" {

pkg/filesystem/driver/onedrive/handler.go

@@ -14,6 +14,7 @@ import (
 	model "github.com/cloudreve/Cloudreve/v3/models"
 	"github.com/cloudreve/Cloudreve/v3/pkg/auth"
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
 	"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -27,6 +28,16 @@ type Driver struct {
 	HTTPClient request.Client
 }
 
+// NewDriver 从存储策略初始化新的Driver实例
+func NewDriver(policy *model.Policy) (driver.Handler, error) {
+	client, err := NewClient(policy)
+	return Driver{
+		Policy:     policy,
+		Client:     client,
+		HTTPClient: request.NewClient(),
+	}, err
+}
+
 // List 列取项目
 func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
 	base = strings.TrimPrefix(base, "/")

pkg/filesystem/driver/onedrive/lock.go

@@ -0,0 +1,25 @@
+package onedrive
+
+import "sync"
+
+// CredentialLock 针对存储策略凭证的锁
+type CredentialLock interface {
+	Lock(uint)
+	Unlock(uint)
+}
+
+var GlobalMutex = mutexMap{}
+
+type mutexMap struct {
+	locks sync.Map
+}
+
+func (m *mutexMap) Lock(id uint) {
+	lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
+	lock.(*sync.Mutex).Lock()
+}
+
+func (m *mutexMap) Unlock(id uint) {
+	lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
+	lock.(*sync.Mutex).Unlock()
+}

pkg/filesystem/driver/onedrive/oauth.go

@@ -10,7 +10,9 @@ import (
 	"time"
 
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/request"
+	"github.com/cloudreve/Cloudreve/v3/pkg/slave"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 )
 
@@ -124,6 +126,13 @@ func (client *Client) ObtainToken(ctx context.Context, opts ...Option) (*Credent
 
 // UpdateCredential 更新凭证，并检查有效期
 func (client *Client) UpdateCredential(ctx context.Context) error {
+	if conf.SystemConfig.Mode == "slave" {
+		return client.fetchCredentialFromMaster(ctx)
+	}
+
+	GlobalMutex.Lock(client.Policy.ID)
+	defer GlobalMutex.Unlock(client.Policy.ID)
+
 	// 如果已存在凭证
 	if client.Credential != nil && client.Credential.AccessToken != "" {
 		// 检查已有凭证是否过期
@@ -160,11 +169,21 @@ func (client *Client) UpdateCredential(ctx context.Context) error {
 	client.Credential = credential
 
 	// 更新存储策略的 RefreshToken
-	client.Policy.AccessKey = credential.RefreshToken
-	client.Policy.SaveAndClearCache()
+	client.Policy.UpdateAccessKeyAndClearCache(credential.RefreshToken)
 
 	// 更新缓存
 	cache.Set("onedrive_"+client.ClientID, *credential, int(expires))
 
 	return nil
 }
+
+// UpdateCredential 更新凭证，并检查有效期
+func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
+	res, err := slave.DefaultController.GetOneDriveToken(client.Policy.MasterID, client.Policy.ID)
+	if err != nil {
+		return err
+	}
+
+	client.Credential = &Credential{AccessToken: res}
+	return nil
+}