wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-26 09:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: aria2 download and transfer in slave node (#1040)

Browse Source 
* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings
This commit is contained in:
AaronLiu
2021-10-31 09:41:56 +08:00
committed by GitHub
parent a3b4a22dbc
commit 056de22edb
74 changed files with 3647 additions and 715 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
service/admin/aria2.go
View File

@@ -1,43 +1,71 @@
package admin
import (
"bytes"
"encoding/json"
model "github.com/cloudreve/Cloudreve/v3/models"
"net/url"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Aria2TestService aria2连接测试服务
type Aria2TestService struct {
Server string `json:"server" binding:"required"`
Token string `json:"token"`
Server string `json:"server" binding:"required"`
RPC string `json:"rpc" binding:"required"`
Secret string `json:"secret" binding:"required"`
Token string `json:"token"`
Type model.ModelType `json:"type"`
}
// Test 测试aria2连接
func (service *Aria2TestService) Test() serializer.Response {
testRPC := aria2.RPCService{}
// 解析RPC服务地址
server, err := url.Parse(service.Server)
func (service *Aria2TestService) TestMaster() serializer.Response {
res, err := aria2.TestRPCConnection(service.RPC, service.Token, 5)
if err != nil {
return serializer.ParamErr("无法解析 aria2 RPC 服务地址, "+err.Error(), nil)
}
server.Path = "/jsonrpc"
if err := testRPC.Init(server.String(), service.Token, 5, map[string]interface{}{}); err != nil {
return serializer.ParamErr("无法初始化连接, "+err.Error(), nil)
return serializer.ParamErr(err.Error(), err)
}
defer testRPC.Caller.Close()
info, err := testRPC.Caller.GetVersion()
if err != nil {
return serializer.ParamErr("无法请求 RPC 服务, "+err.Error(), nil)
}
if info.Version == "" {
if res.Version == "" {
return serializer.ParamErr("RPC 服务返回非预期响应", nil)
}
return serializer.Response{Data: info.Version}
return serializer.Response{Data: res.Version}
}
func (service *Aria2TestService) TestSlave() serializer.Response {
slave, err := url.Parse(service.Server)
if err != nil {
return serializer.ParamErr("无法解析从机端地址，"+err.Error(), nil)
}
controller, _ := url.Parse("/api/v3/slave/ping/aria2")
// 请求正文
service.Type = model.MasterNodeType
bodyByte, _ := json.Marshal(service)
r := request.NewClient()
res, err := r.Request(
"POST",
slave.ResolveReference(controller).String(),
bytes.NewReader(bodyByte),
request.WithTimeout(time.Duration(10)*time.Second),
request.WithCredential(
auth.HMACAuth{SecretKey: []byte(service.Secret)},
int64(model.GetIntSetting("slave_api_timeout", 60)),
),
).DecodeResponse()
if err != nil {
return serializer.ParamErr("无连接到从机，"+err.Error(), nil)
}
if res.Code != 0 {
return serializer.ParamErr("成功接到从机，但是从机返回："+res.Msg, nil)
}
return serializer.Response{Data: res.Data.(string)}
}

138
service/admin/node.go Normal file
View File

@@ -0,0 +1,138 @@
package admin
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"strings"
)
// AddNodeService 节点添加服务
type AddNodeService struct {
Node model.Node `json:"node" binding:"required"`
}
// Add 添加节点
func (service *AddNodeService) Add() serializer.Response {
if service.Node.ID > 0 {
if err := model.DB.Save(&service.Node).Error; err != nil {
return serializer.ParamErr("节点保存失败", err)
}
} else {
if err := model.DB.Create(&service.Node).Error; err != nil {
return serializer.ParamErr("节点添加失败", err)
}
}
return serializer.Response{Data: service.Node.ID}
}
// Nodes 列出从机节点
func (service *AdminListService) Nodes() serializer.Response {
var res []model.Node
total := 0
tx := model.DB.Model(&model.Node{})
if service.OrderBy != "" {
tx = tx.Order(service.OrderBy)
}
for k, v := range service.Conditions {
tx = tx.Where(k+" = ?", v)
}
if len(service.Searches) > 0 {
search := ""
for k, v := range service.Searches {
search += k + " like '%" + v + "%' OR "
}
search = strings.TrimSuffix(search, " OR ")
tx = tx.Where(search)
}
// 计算总数用于分页
tx.Count(&total)
// 查询记录
tx.Limit(service.PageSize).Offset((service.Page - 1) * service.PageSize).Find(&res)
isActive := make(map[uint]bool)
for i := 0; i < len(res); i++ {
if node := cluster.Default.GetNodeByID(res[i].ID); node != nil {
isActive[res[i].ID] = node.IsActive()
}
}
return serializer.Response{Data: map[string]interface{}{
"total": total,
"items": res,
"active": isActive,
}}
}
// ToggleNodeService 开关节点服务
type ToggleNodeService struct {
ID uint `uri:"id"`
Desired model.NodeStatus `uri:"desired"`
}
// Toggle 开关节点
func (service *ToggleNodeService) Toggle() serializer.Response {
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.DBErr("找不到节点", err)
}
// 是否为系统节点
if node.ID <= 1 {
return serializer.Err(serializer.CodeNoPermissionErr, "系统节点无法更改", err)
}
if err = node.SetStatus(service.Desired); err != nil {
return serializer.DBErr("无法更改节点状态", err)
}
if service.Desired == model.NodeActive {
cluster.Default.Add(&node)
} else {
cluster.Default.Delete(node.ID)
}
return serializer.Response{}
}
// NodeService 节点ID服务
type NodeService struct {
ID uint `uri:"id" json:"id" binding:"required"`
}
// Delete 删除节点
func (service *NodeService) Delete() serializer.Response {
// 查找用户组
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "节点不存在", err)
}
// 是否为系统节点
if node.ID <= 1 {
return serializer.Err(serializer.CodeNoPermissionErr, "系统节点无法删除", err)
}
cluster.Default.Delete(node.ID)
if err := model.DB.Delete(&node).Error; err != nil {
return serializer.DBErr("无法删除节点", err)
}
return serializer.Response{}
}
// Get 获取节点详情
func (service *NodeService) Get() serializer.Response {
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "节点不存在", err)
}
return serializer.Response{Data: node}
}

10
service/admin/policy.go
View File

@@ -151,7 +151,7 @@ func (service *PolicyService) AddCORS() serializer.Response {
case "oss":
handler := oss.Driver{
Policy: &policy,
HTTPClient: request.HTTPClient{},
HTTPClient: request.NewClient(),
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
@@ -161,7 +161,7 @@ func (service *PolicyService) AddCORS() serializer.Response {
b := &cossdk.BaseURL{BucketURL: u}
handler := cos.Driver{
Policy: &policy,
HTTPClient: request.HTTPClient{},
HTTPClient: request.NewClient(),
Client: cossdk.NewClient(b, &http.Client{
Transport: &cossdk.AuthorizationTransport{
SecretID: policy.AccessKey,
@@ -195,7 +195,7 @@ func (service *SlavePingService) Test() serializer.Response {
controller, _ := url.Parse("/api/v3/site/ping")
r := request.HTTPClient{}
r := request.NewClient()
res, err := r.Request(
"GET",
master.ResolveReference(controller).String(),
@@ -229,7 +229,7 @@ func (service *SlaveTestService) Test() serializer.Response {
}
bodyByte, _ := json.Marshal(body)
r := request.HTTPClient{}
r := request.NewClient()
res, err := r.Request(
"POST",
slave.ResolveReference(controller).String(),
@@ -245,7 +245,7 @@ func (service *SlaveTestService) Test() serializer.Response {
}
if res.Code != 0 {
return serializer.ParamErr("成功接到从机，但是"+res.Msg, nil)
return serializer.ParamErr("成功接到从机，但是从机返回："+res.Msg, nil)
}
return serializer.Response{}