Feat: aria2 download and transfer in slave node (#1040)

* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings

pkg/serializer/auth.go

@@ -5,16 +5,15 @@ import "encoding/json"
 // RequestRawSign 待签名的HTTP请求
 type RequestRawSign struct {
 	Path   string
-	Policy string
+	Header string
 	Body   string
 }
 
 // NewRequestSignString 返回JSON格式的待签名字符串
-// TODO 测试
-func NewRequestSignString(path, policy, body string) string {
+func NewRequestSignString(path, header, body string) string {
 	req := RequestRawSign{
 		Path:   path,
-		Policy: policy,
+		Header: header,
 		Body:   body,
 	}
 	res, _ := json.Marshal(req)

pkg/serializer/error.go

@@ -1,14 +1,9 @@
 package serializer
 
-import "github.com/gin-gonic/gin"
-
-// Response 基础序列化器
-type Response struct {
-	Code  int         `json:"code"`
-	Data  interface{} `json:"data,omitempty"`
-	Msg   string      `json:"msg"`
-	Error string      `json:"error,omitempty"`
-}
+import (
+	"errors"
+	"github.com/gin-gonic/gin"
+)
 
 // AppError 应用错误，实现了error接口
 type AppError struct {
@@ -17,7 +12,7 @@ type AppError struct {
 	RawError error
 }
 
-// NewError 返回新的错误对象 todo:测试 还有下面的
+// NewError 返回新的错误对象
 func NewError(code int, msg string, err error) AppError {
 	return AppError{
 		Code:     code,
@@ -26,6 +21,15 @@ func NewError(code int, msg string, err error) AppError {
 	}
 }
 
+// NewErrorFromResponse 从 serializer.Response 构建错误
+func NewErrorFromResponse(resp *Response) AppError {
+	return AppError{
+		Code:     resp.Code,
+		Msg:      resp.Msg,
+		RawError: errors.New(resp.Error),
+	}
+}
+
 // WithError 将应用error携带标准库中的error
 func (err *AppError) WithError(raw error) AppError {
 	err.RawError = raw
@@ -66,6 +70,8 @@ const (
 	CodeGroupNotAllowed = 40007
 	// CodeAdminRequired 非管理用户组
 	CodeAdminRequired = 40008
+	// CodeMasterNotFound 主机节点未注册
+	CodeMasterNotFound = 40009
 	// CodeDBError 数据库操作失败
 	CodeDBError = 50001
 	// CodeEncryptError 加密失败

pkg/serializer/response.go

@@ -0,0 +1,35 @@
+package serializer
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/gob"
+)
+
+// Response 基础序列化器
+type Response struct {
+	Code  int         `json:"code"`
+	Data  interface{} `json:"data,omitempty"`
+	Msg   string      `json:"msg"`
+	Error string      `json:"error,omitempty"`
+}
+
+// NewResponseWithGobData 返回Data字段使用gob编码的Response
+func NewResponseWithGobData(data interface{}) Response {
+	var w bytes.Buffer
+	encoder := gob.NewEncoder(&w)
+	if err := encoder.Encode(data); err != nil {
+		return Err(CodeInternalSetting, "无法编码返回结果", err)
+	}
+
+	return Response{Data: w.Bytes()}
+}
+
+// GobDecode 将 Response 正文解码至目标指针
+func (r *Response) GobDecode(target interface{}) {
+	src := r.Data.(string)
+	raw := make([]byte, len(src)*len(src)/base64.StdEncoding.DecodedLen(len(src)))
+	base64.StdEncoding.Decode(raw, []byte(src))
+	decoder := gob.NewDecoder(bytes.NewBuffer(raw))
+	decoder.Decode(target)
+}

pkg/serializer/slave.go

@@ -1,5 +1,12 @@
 package serializer
 
+import (
+	"crypto/sha1"
+	"encoding/gob"
+	"fmt"
+	model "github.com/cloudreve/Cloudreve/v3/models"
+)
+
 // RemoteDeleteRequest 远程策略删除接口请求正文
 type RemoteDeleteRequest struct {
 	Files []string `json:"files"`
@@ -10,3 +17,51 @@ type ListRequest struct {
 	Path      string `json:"path"`
 	Recursive bool   `json:"recursive"`
 }
+
+// NodePingReq 从机节点Ping请求
+type NodePingReq struct {
+	SiteURL       string      `json:"site_url"`
+	SiteID        string      `json:"site_id"`
+	IsUpdate      bool        `json:"is_update"`
+	CredentialTTL int         `json:"credential_ttl"`
+	Node          *model.Node `json:"node"`
+}
+
+// NodePingResp 从机节点Ping响应
+type NodePingResp struct {
+}
+
+// SlaveAria2Call 从机有关Aria2的请求正文
+type SlaveAria2Call struct {
+	Task         *model.Download        `json:"task"`
+	GroupOptions map[string]interface{} `json:"group_options"`
+	Files        []int                  `json:"files"`
+}
+
+// SlaveTransferReq 从机中转任务创建请求
+type SlaveTransferReq struct {
+	Src    string        `json:"src"`
+	Dst    string        `json:"dst"`
+	Policy *model.Policy `json:"policy"`
+}
+
+// Hash 返回创建请求的唯一标识，保持创建请求幂等
+func (s *SlaveTransferReq) Hash(id string) string {
+	h := sha1.New()
+	h.Write([]byte(fmt.Sprintf("transfer-%s-%s-%s-%d", id, s.Src, s.Dst, s.Policy.ID)))
+	bs := h.Sum(nil)
+	return fmt.Sprintf("%x", bs)
+}
+
+const (
+	SlaveTransferSuccess = "success"
+	SlaveTransferFailed  = "failed"
+)
+
+type SlaveTransferResult struct {
+	Error string
+}
+
+func init() {
+	gob.Register(SlaveTransferResult{})
+}