wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-26 09:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: aria2 download and transfer in slave node (#1040)

Browse Source 
* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings
This commit is contained in:
AaronLiu
2021-10-31 09:41:56 +08:00
committed by GitHub
parent a3b4a22dbc
commit 056de22edb
74 changed files with 3647 additions and 715 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
routers/controllers/admin.go
View File

@@ -3,6 +3,7 @@ package controllers
import (
"io"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/email"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
@@ -24,7 +25,7 @@ func AdminSummary(c *gin.Context) {
// AdminNews 获取社区新闻
func AdminNews(c *gin.Context) {
r := request.HTTPClient{}
r := request.NewClient()
res := r.Request("GET", "https://forum.cloudreve.org/api/discussions?include=startUser%2ClastUser%2CstartPost%2Ctags&filter%5Bq%5D=%20tag%3Anotice&sort=-startTime&page%5Blimit%5D=10", nil)
if res.Err == nil {
io.Copy(c.Writer, res.Response.Body)
@@ -92,7 +93,13 @@ func AdminSendTestMail(c *gin.Context) {
func AdminTestAria2(c *gin.Context) {
var service admin.Aria2TestService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Test()
var res serializer.Response
if service.Type == model.MasterNodeType {
res = service.TestMaster()
} else {
res = service.TestSlave()
}
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
@@ -425,3 +432,58 @@ func AdminListFolders(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// AdminListNodes 列出从机节点
func AdminListNodes(c *gin.Context) {
var service admin.AdminListService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Nodes()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminAddNode 新建节点
func AdminAddNode(c *gin.Context) {
var service admin.AddNodeService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Add()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminToggleNode 启用/暂停节点
func AdminToggleNode(c *gin.Context) {
var service admin.ToggleNodeService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Toggle()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminDeleteGroup 删除用户组
func AdminDeleteNode(c *gin.Context) {
var service admin.NodeService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Delete()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminGetNode 获取节点详情
func AdminGetNode(c *gin.Context) {
var service admin.NodeService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Get()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

6
routers/controllers/aria2.go
View File

@@ -3,7 +3,7 @@ package controllers
import (
"context"
ariaCall "github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/service/aria2"
"github.com/cloudreve/Cloudreve/v3/service/explorer"
"github.com/gin-gonic/gin"
@@ -13,7 +13,7 @@ import (
func AddAria2URL(c *gin.Context) {
var addService aria2.AddURLService
if err := c.ShouldBindJSON(&addService); err == nil {
res := addService.Add(c, ariaCall.URLTask)
res := addService.Add(c, common.URLTask)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
@@ -52,7 +52,7 @@ func AddAria2Torrent(c *gin.Context) {
if err := c.ShouldBindJSON(&addService); err == nil {
addService.URL = res.Data.(string)
res := addService.Add(c, ariaCall.URLTask)
res := addService.Add(c, common.URLTask)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))

101
routers/controllers/slave.go
View File

@@ -10,7 +10,9 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/service/admin"
"github.com/cloudreve/Cloudreve/v3/service/aria2"
"github.com/cloudreve/Cloudreve/v3/service/explorer"
"github.com/cloudreve/Cloudreve/v3/service/node"
"github.com/gin-gonic/gin"
)
@@ -175,3 +177,102 @@ func SlaveList(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveHeartbeat 接受主机心跳包
func SlaveHeartbeat(c *gin.Context) {
var service serializer.NodePingReq
if err := c.ShouldBindJSON(&service); err == nil {
res := node.HandleMasterHeartbeat(&service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveAria2Create 创建 Aria2 任务
func SlaveAria2Create(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.Add(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveAria2Status 查询从机 Aria2 任务状态
func SlaveAria2Status(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveStatus(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveCancelAria2Task 取消从机离线下载任务
func SlaveCancelAria2Task(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveCancel(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveSelectTask 从机选取离线下载文件
func SlaveSelectTask(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveSelect(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveCreateTransferTask 从机创建中转任务
func SlaveCreateTransferTask(c *gin.Context) {
var service serializer.SlaveTransferReq
if err := c.ShouldBindJSON(&service); err == nil {
res := explorer.CreateTransferTask(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveNotificationPush 处理从机发送的消息推送
func SlaveNotificationPush(c *gin.Context) {
var service node.SlaveNotificationService
if err := c.ShouldBindUri(&service); err == nil {
res := service.HandleSlaveNotificationPush(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveGetOneDriveCredential 从机获取主机的OneDrive存储策略凭证
func SlaveGetOneDriveCredential(c *gin.Context) {
var service node.OneDriveCredentialService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Get(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveSelectTask 从机删除离线下载临时文件
func SlaveDeleteTempFile(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveDeleteTemp(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

63
routers/router.go
View File

@@ -2,6 +2,7 @@ package routers
import (
"github.com/cloudreve/Cloudreve/v3/middleware"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/hashid"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
@@ -29,7 +30,9 @@ func InitSlaveRouter() *gin.Engine {
InitCORS(r)
v3 := r.Group("/api/v3/slave")
// 鉴权中间件
v3.Use(middleware.SignRequired())
v3.Use(middleware.SignRequired(auth.General))
// 主机信息解析
v3.Use(middleware.MasterMetadata())
/*
路由
@@ -37,6 +40,10 @@ func InitSlaveRouter() *gin.Engine {
{
// Ping
v3.POST("ping", controllers.SlavePing)
// 测试 Aria2 RPC 连接
v3.POST("ping/aria2", controllers.AdminTestAria2)
// 接收主机心跳包
v3.POST("heartbeat", controllers.SlaveHeartbeat)
// 上传
v3.POST("upload", controllers.SlaveUpload)
// 下载
@@ -49,6 +56,28 @@ func InitSlaveRouter() *gin.Engine {
v3.POST("delete", controllers.SlaveDelete)
// 列出文件
v3.POST("list", controllers.SlaveList)
// 离线下载
aria2 := v3.Group("aria2")
aria2.Use(middleware.UseSlaveAria2Instance())
{
// 创建离线下载任务
aria2.POST("task", controllers.SlaveAria2Create)
// 获取任务状态
aria2.POST("status", controllers.SlaveAria2Status)
// 取消离线下载任务
aria2.POST("cancel", controllers.SlaveCancelAria2Task)
// 选取任务文件
aria2.POST("select", controllers.SlaveSelectTask)
// 删除任务临时文件
aria2.POST("delete", controllers.SlaveDeleteTempFile)
}
// 异步任务
task := v3.Group("task")
{
task.PUT("transfer", controllers.SlaveCreateTransferTask)
}
}
return r
}
@@ -131,7 +160,7 @@ func InitMasterRouter() *gin.Engine {
user.PATCH("reset", controllers.UserReset)
// 邮件激活
user.GET("activate/:id",
middleware.SignRequired(),
middleware.SignRequired(auth.General),
middleware.HashID(hashid.UserID),
controllers.UserActivate,
)
@@ -159,7 +188,7 @@ func InitMasterRouter() *gin.Engine {
// 需要携带签名验证的
sign := v3.Group("")
sign.Use(middleware.SignRequired())
sign.Use(middleware.SignRequired(auth.General))
{
file := sign.Group("file")
{
@@ -174,6 +203,18 @@ func InitMasterRouter() *gin.Engine {
}
}
// 从机的 RPC 通信
slave := v3.Group("slave")
slave.Use(middleware.SlaveRPCSignRequired())
{
// 事件通知
slave.PUT("notification/:subject", controllers.SlaveNotificationPush)
// 上传
slave.POST("upload", controllers.SlaveUpload)
// OneDrive 存储策略凭证
slave.GET("credential/onedrive/:id", controllers.SlaveGetOneDriveCredential)
}
// 回调接口
callback := v3.Group("callback")
{
@@ -405,6 +446,22 @@ func InitMasterRouter() *gin.Engine {
task.POST("import", controllers.AdminCreateImportTask)
}
node := admin.Group("node")
{
// 列出从机节点
node.POST("list", controllers.AdminListNodes)
// 列出从机节点
node.POST("aria2/test", controllers.AdminTestAria2)
// 创建/保存节点
node.POST("", controllers.AdminAddNode)
// 启用/暂停节点
node.PATCH("enable/:id/:desired", controllers.AdminToggleNode)
// 删除节点
node.DELETE(":id", controllers.AdminDeleteNode)
// 获取节点
node.GET(":id", controllers.AdminGetNode)
}
}
// 用户