wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-26 09:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: aria2 download and transfer in slave node (#1040)

Browse Source 
* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings
This commit is contained in:
AaronLiu
2021-10-31 09:41:56 +08:00
committed by GitHub
parent a3b4a22dbc
commit 056de22edb
74 changed files with 3647 additions and 715 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
service/aria2/add.go
View File

@@ -3,8 +3,14 @@ package aria2
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/monitor"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/slave"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
@@ -14,7 +20,7 @@ type AddURLService struct {
Dst string `json:"dst" binding:"required,min=1"`
}
// Add 创建新的链接离线下载任务
// Add 主机创建新的链接离线下载任务
func (service *AddURLService) Add(c *gin.Context, taskType int) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
@@ -35,19 +41,60 @@ func (service *AddURLService) Add(c *gin.Context, taskType int) serializer.Respo
// 创建任务
task := &model.Download{
Status: aria2.Ready,
Status: common.Ready,
Type: taskType,
Dst: service.Dst,
UserID: fs.User.ID,
Source: service.URL,
}
// 获取 Aria2 负载均衡器
aria2.Lock.RLock()
if err := aria2.Instance.CreateTask(task, fs.User.Group.OptionsSerialized.Aria2Options); err != nil {
aria2.Lock.RUnlock()
lb := aria2.LB
aria2.Lock.RUnlock()
// 获取 Aria2 实例
err, node := cluster.Default.BalanceNodeByFeature("aria2", lb)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Aria2 实例获取失败", err)
}
// 创建任务
gid, err := node.GetAria2Instance().CreateTask(task, fs.User.Group.OptionsSerialized.Aria2Options)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "任务创建失败", err)
}
aria2.Lock.RUnlock()
task.GID = gid
task.NodeID = node.ID()
_, err = task.Create()
if err != nil {
return serializer.DBErr("任务创建失败", err)
}
// 创建任务监控
monitor.NewMonitor(task)
return serializer.Response{}
}
// Add 从机创建新的链接离线下载任务
func Add(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 创建任务
gid, err := caller.(common.Aria2).CreateTask(service.Task, service.GroupOptions)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "无法创建离线下载任务", err)
}
// 创建事件通知回调
siteID, _ := c.Get("MasterSiteID")
mq.GlobalMQ.SubscribeCallback(gid, func(message mq.Message) {
if err := slave.DefaultController.SendNotification(siteID.(string), message.TriggeredBy, message); err != nil {
util.Log().Warning("无法发送离线下载任务状态变更通知, %s", err)
}
})
return serializer.Response{Data: gid}
}

81
service/aria2/manage.go
View File

@@ -2,7 +2,8 @@ package aria2
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
)
@@ -25,14 +26,14 @@ type DownloadListService struct {
// Finished 获取已完成的任务
func (service *DownloadListService) Finished(c *gin.Context, user *model.User) serializer.Response {
// 查找下载记录
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, aria2.Error, aria2.Complete, aria2.Canceled, aria2.Unknown)
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, common.Error, common.Complete, common.Canceled, common.Unknown)
return serializer.BuildFinishedListResponse(downloads)
}
// Downloading 获取正在下载中的任务
func (service *DownloadListService) Downloading(c *gin.Context, user *model.User) serializer.Response {
// 查找下载记录
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, aria2.Downloading, aria2.Paused, aria2.Ready)
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, common.Downloading, common.Paused, common.Ready)
return serializer.BuildDownloadingResponse(downloads)
}
@@ -47,7 +48,7 @@ func (service *DownloadTaskService) Delete(c *gin.Context) serializer.Response {
return serializer.Err(serializer.CodeNotFound, "下载记录不存在", err)
}
if download.Status >= aria2.Error {
if download.Status >= common.Error {
// 如果任务已完成，则删除任务记录
if err := download.Delete(); err != nil {
return serializer.Err(serializer.CodeDBError, "任务记录删除失败", err)
@@ -56,9 +57,12 @@ func (service *DownloadTaskService) Delete(c *gin.Context) serializer.Response {
}
// 取消任务
aria2.Lock.RLock()
defer aria2.Lock.RUnlock()
if err := aria2.Instance.Cancel(download); err != nil {
node := cluster.Default.GetNodeByID(download.GetNodeID())
if node == nil {
return serializer.Err(serializer.CodeInternalSetting, "目标节点不可用", err)
}
if err := node.GetAria2Instance().Cancel(download); err != nil {
return serializer.Err(serializer.CodeNotSet, "操作失败", err)
}
@@ -76,17 +80,72 @@ func (service *SelectFileService) Select(c *gin.Context) serializer.Response {
return serializer.Err(serializer.CodeNotFound, "下载记录不存在", err)
}
if download.StatusInfo.BitTorrent.Mode != "multi" || (download.Status != aria2.Downloading && download.Status != aria2.Paused) {
if download.StatusInfo.BitTorrent.Mode != "multi" || (download.Status != common.Downloading && download.Status != common.Paused) {
return serializer.Err(serializer.CodeNoPermissionErr, "此下载任务无法选取文件", err)
}
// 选取下载
aria2.Lock.RLock()
defer aria2.Lock.RUnlock()
if err := aria2.Instance.Select(download, service.Indexes); err != nil {
node := cluster.Default.GetNodeByID(download.GetNodeID())
if err := node.GetAria2Instance().Select(download, service.Indexes); err != nil {
return serializer.Err(serializer.CodeNotSet, "操作失败", err)
}
return serializer.Response{}
}
// SlaveStatus 从机查询离线任务状态
func SlaveStatus(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
status, err := caller.(common.Aria2).Status(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "离线下载任务查询失败", err)
}
return serializer.NewResponseWithGobData(status)
}
// SlaveCancel 取消从机离线下载任务
func SlaveCancel(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).Cancel(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "任务取消失败", err)
}
return serializer.Response{}
}
// SlaveSelect 从机选取离线下载任务文件
func SlaveSelect(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).Select(service.Task, service.Files)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "任务选取失败", err)
}
return serializer.Response{}
}
// SlaveSelect 从机选取离线下载任务文件
func SlaveDeleteTemp(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).DeleteTempFile(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "临时文件删除失败", err)
}
return serializer.Response{}
}