Feat: aria2 download and transfer in slave node (#1040)

* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings

models/download.go

@@ -24,6 +24,7 @@ type Download struct {
 	Dst            string `gorm:"type:text"`       // 用户文件系统存储父目录路径
 	UserID         uint   // 发起者UID
 	TaskID         uint   // 对应的转存任务ID
+	NodeID         uint   // 处理任务的节点ID
 
 	// 关联模型
 	User *User `gorm:"PRELOAD:false,association_autoupdate:false"`
@@ -114,3 +115,13 @@ func (task *Download) GetOwner() *User {
 func (download *Download) Delete() error {
 	return DB.Model(download).Delete(download).Error
 }
+
+// GetNodeID 返回任务所属节点ID
+func (task *Download) GetNodeID() uint {
+	// 兼容3.4版本之前生成的下载记录
+	if task.NodeID == 0 {
+		return 1
+	}
+
+	return task.NodeID
+}

models/migration.go

@@ -5,6 +5,7 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 	"github.com/fatih/color"
+	"github.com/gofrs/uuid"
 	"github.com/jinzhu/gorm"
 )
 
@@ -34,8 +35,9 @@ func migration() {
 	if conf.DatabaseConfig.Type == "mysql" {
 		DB = DB.Set("gorm:table_options", "ENGINE=InnoDB")
 	}
+
 	DB.AutoMigrate(&User{}, &Setting{}, &Group{}, &Policy{}, &Folder{}, &File{}, &Share{},
-		&Task{}, &Download{}, &Tag{}, &Webdav{})
+		&Task{}, &Download{}, &Tag{}, &Webdav{}, &Node{})
 
 	// 创建初始存储策略
 	addDefaultPolicy()
@@ -73,6 +75,8 @@ func addDefaultPolicy() {
 }
 
 func addDefaultSettings() {
+	siteID, _ := uuid.NewV4()
+
 	defaultSettings := []Setting{
 		{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
 		{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
@@ -83,6 +87,7 @@ func addDefaultSettings() {
 		{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
 		{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
 		{Name: "siteScript", Value: ``, Type: "basic"},
+		{Name: "siteID", Value: siteID.String(), Type: "basic"},
 		{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
 		{Name: "mail_keepalive", Value: `30`, Type: "mail"},
 		{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
@@ -100,10 +105,13 @@ func addDefaultSettings() {
 		{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
 		{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
 		{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
+		{Name: "slave_node_retry", Value: `3`, Type: "slave"},
+		{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
+		{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
+		{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
 		{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
 		{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
 		{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
-		{Name: "aria2_call_timeout", Value: `5`, Type: "timeout"},
 		{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
 		{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
 		{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
@@ -131,11 +139,6 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
 		{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
 		{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
 		{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
-		{Name: "aria2_token", Value: ``, Type: "aria2"},
-		{Name: "aria2_rpcurl", Value: ``, Type: "aria2"},
-		{Name: "aria2_temp_path", Value: ``, Type: "aria2"},
-		{Name: "aria2_options", Value: `{}`, Type: "aria2"},
-		{Name: "aria2_interval", Value: `60`, Type: "aria2"},
 		{Name: "max_worker_num", Value: `10`, Type: "task"},
 		{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
 		{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},

models/node.go

@@ -0,0 +1,91 @@
+package model
+
+import (
+	"encoding/json"
+	"github.com/jinzhu/gorm"
+)
+
+// Node 从机节点信息模型
+type Node struct {
+	gorm.Model
+	Status       NodeStatus // 节点状态
+	Name         string     // 节点别名
+	Type         ModelType  // 节点状态
+	Server       string     // 服务器地址
+	SlaveKey     string     `gorm:"type:text"` // 主->从 通信密钥
+	MasterKey    string     `gorm:"type:text"` // 从->主 通信密钥
+	Aria2Enabled bool       // 是否支持用作离线下载节点
+	Aria2Options string     `gorm:"type:text"` // 离线下载配置
+	Rank         int        // 负载均衡权重
+
+	// 数据库忽略字段
+	Aria2OptionsSerialized Aria2Option `gorm:"-"`
+}
+
+// Aria2Option 非公有的Aria2配置属性
+type Aria2Option struct {
+	// RPC 服务器地址
+	Server string `json:"server,omitempty"`
+	// RPC 密钥
+	Token string `json:"token,omitempty"`
+	// 临时下载目录
+	TempPath string `json:"temp_path,omitempty"`
+	// 附加下载配置
+	Options string `json:"options,omitempty"`
+	// 下载监控间隔
+	Interval int `json:"interval,omitempty"`
+	// RPC API 请求超时
+	Timeout int `json:"timeout,omitempty"`
+}
+
+type NodeStatus int
+type ModelType int
+
+const (
+	NodeActive NodeStatus = iota
+	NodeSuspend
+)
+
+const (
+	SlaveNodeType ModelType = iota
+	MasterNodeType
+)
+
+// GetNodeByID 用ID获取节点
+func GetNodeByID(ID interface{}) (Node, error) {
+	var node Node
+	result := DB.First(&node, ID)
+	return node, result.Error
+}
+
+// GetNodesByStatus 根据给定状态获取节点
+func GetNodesByStatus(status ...NodeStatus) ([]Node, error) {
+	var nodes []Node
+	result := DB.Where("status in (?)", status).Find(&nodes)
+	return nodes, result.Error
+}
+
+// AfterFind 找到节点后的钩子
+func (node *Node) AfterFind() (err error) {
+	// 解析离线下载设置到 Aria2OptionsSerialized
+	if node.Aria2Options != "" {
+		err = json.Unmarshal([]byte(node.Aria2Options), &node.Aria2OptionsSerialized)
+	}
+
+	return err
+}
+
+// BeforeSave Save策略前的钩子
+func (node *Node) BeforeSave() (err error) {
+	optionsValue, err := json.Marshal(&node.Aria2OptionsSerialized)
+	node.Aria2Options = string(optionsValue)
+	return err
+}
+
+// SetStatus 设置节点启用状态
+func (node *Node) SetStatus(status NodeStatus) error {
+	node.Status = status
+	return DB.Model(node).Updates(map[string]interface{}{
+		"status": status,
+	}).Error
+}

models/policy.go

@@ -37,6 +37,7 @@ type Policy struct {
 
 	// 数据库忽略字段
 	OptionsSerialized PolicyOption `gorm:"-"`
+	MasterID          string       `gorm:"-"`
 }
 
 // PolicyOption 非公有的存储策略属性
@@ -277,6 +278,13 @@ func (policy *Policy) SaveAndClearCache() error {
 	return err
 }
 
+// SaveAndClearCache 更新并清理缓存
+func (policy *Policy) UpdateAccessKeyAndClearCache(s string) error {
+	err := DB.Model(policy).UpdateColumn("access_key", s).Error
+	policy.ClearCache()
+	return err
+}
+
 // ClearCache 清空policy缓存
 func (policy *Policy) ClearCache() {
 	cache.Deletes([]string{strconv.FormatUint(uint64(policy.ID), 10)}, "policy_")

models/setting.go

@@ -30,12 +30,16 @@ func GetSettingByName(name string) string {
 	if optionValue, ok := cache.Get(cacheKey); ok {
 		return optionValue.(string)
 	}
+
 	// 尝试数据库中查找
-	result := DB.Where("name = ?", name).First(&setting)
-	if result.Error == nil {
-		_ = cache.Set(cacheKey, setting.Value, -1)
-		return setting.Value
+	if DB != nil {
+		result := DB.Where("name = ?", name).First(&setting)
+		if result.Error == nil {
+			_ = cache.Set(cacheKey, setting.Value, -1)
+			return setting.Value
+		}
 	}
+
 	return ""
 }