wsy182/Cloudreve
1
0
Fork 0
mirror of https://github.com/halejohn/Cloudreve.git synced 2026-01-26 09:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: aria2 download and transfer in slave node (#1040)

Browse Source 
* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings
This commit is contained in:
AaronLiu
2021-10-31 09:41:56 +08:00
committed by GitHub
parent a3b4a22dbc
commit 056de22edb
74 changed files with 3647 additions and 715 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
service/admin/aria2.go
View File

@@ -1,43 +1,71 @@
package admin
import (
"bytes"
"encoding/json"
model "github.com/cloudreve/Cloudreve/v3/models"
"net/url"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Aria2TestService aria2连接测试服务
type Aria2TestService struct {
Server string `json:"server" binding:"required"`
Token string `json:"token"`
Server string `json:"server" binding:"required"`
RPC string `json:"rpc" binding:"required"`
Secret string `json:"secret" binding:"required"`
Token string `json:"token"`
Type model.ModelType `json:"type"`
}
// Test 测试aria2连接
func (service *Aria2TestService) Test() serializer.Response {
testRPC := aria2.RPCService{}
// 解析RPC服务地址
server, err := url.Parse(service.Server)
func (service *Aria2TestService) TestMaster() serializer.Response {
res, err := aria2.TestRPCConnection(service.RPC, service.Token, 5)
if err != nil {
return serializer.ParamErr("无法解析 aria2 RPC 服务地址, "+err.Error(), nil)
}
server.Path = "/jsonrpc"
if err := testRPC.Init(server.String(), service.Token, 5, map[string]interface{}{}); err != nil {
return serializer.ParamErr("无法初始化连接, "+err.Error(), nil)
return serializer.ParamErr(err.Error(), err)
}
defer testRPC.Caller.Close()
info, err := testRPC.Caller.GetVersion()
if err != nil {
return serializer.ParamErr("无法请求 RPC 服务, "+err.Error(), nil)
}
if info.Version == "" {
if res.Version == "" {
return serializer.ParamErr("RPC 服务返回非预期响应", nil)
}
return serializer.Response{Data: info.Version}
return serializer.Response{Data: res.Version}
}
func (service *Aria2TestService) TestSlave() serializer.Response {
slave, err := url.Parse(service.Server)
if err != nil {
return serializer.ParamErr("无法解析从机端地址，"+err.Error(), nil)
}
controller, _ := url.Parse("/api/v3/slave/ping/aria2")
// 请求正文
service.Type = model.MasterNodeType
bodyByte, _ := json.Marshal(service)
r := request.NewClient()
res, err := r.Request(
"POST",
slave.ResolveReference(controller).String(),
bytes.NewReader(bodyByte),
request.WithTimeout(time.Duration(10)*time.Second),
request.WithCredential(
auth.HMACAuth{SecretKey: []byte(service.Secret)},
int64(model.GetIntSetting("slave_api_timeout", 60)),
),
).DecodeResponse()
if err != nil {
return serializer.ParamErr("无连接到从机，"+err.Error(), nil)
}
if res.Code != 0 {
return serializer.ParamErr("成功接到从机，但是从机返回："+res.Msg, nil)
}
return serializer.Response{Data: res.Data.(string)}
}

138
service/admin/node.go Normal file
View File

@@ -0,0 +1,138 @@
package admin
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"strings"
)
// AddNodeService 节点添加服务
type AddNodeService struct {
Node model.Node `json:"node" binding:"required"`
}
// Add 添加节点
func (service *AddNodeService) Add() serializer.Response {
if service.Node.ID > 0 {
if err := model.DB.Save(&service.Node).Error; err != nil {
return serializer.ParamErr("节点保存失败", err)
}
} else {
if err := model.DB.Create(&service.Node).Error; err != nil {
return serializer.ParamErr("节点添加失败", err)
}
}
return serializer.Response{Data: service.Node.ID}
}
// Nodes 列出从机节点
func (service *AdminListService) Nodes() serializer.Response {
var res []model.Node
total := 0
tx := model.DB.Model(&model.Node{})
if service.OrderBy != "" {
tx = tx.Order(service.OrderBy)
}
for k, v := range service.Conditions {
tx = tx.Where(k+" = ?", v)
}
if len(service.Searches) > 0 {
search := ""
for k, v := range service.Searches {
search += k + " like '%" + v + "%' OR "
}
search = strings.TrimSuffix(search, " OR ")
tx = tx.Where(search)
}
// 计算总数用于分页
tx.Count(&total)
// 查询记录
tx.Limit(service.PageSize).Offset((service.Page - 1) * service.PageSize).Find(&res)
isActive := make(map[uint]bool)
for i := 0; i < len(res); i++ {
if node := cluster.Default.GetNodeByID(res[i].ID); node != nil {
isActive[res[i].ID] = node.IsActive()
}
}
return serializer.Response{Data: map[string]interface{}{
"total": total,
"items": res,
"active": isActive,
}}
}
// ToggleNodeService 开关节点服务
type ToggleNodeService struct {
ID uint `uri:"id"`
Desired model.NodeStatus `uri:"desired"`
}
// Toggle 开关节点
func (service *ToggleNodeService) Toggle() serializer.Response {
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.DBErr("找不到节点", err)
}
// 是否为系统节点
if node.ID <= 1 {
return serializer.Err(serializer.CodeNoPermissionErr, "系统节点无法更改", err)
}
if err = node.SetStatus(service.Desired); err != nil {
return serializer.DBErr("无法更改节点状态", err)
}
if service.Desired == model.NodeActive {
cluster.Default.Add(&node)
} else {
cluster.Default.Delete(node.ID)
}
return serializer.Response{}
}
// NodeService 节点ID服务
type NodeService struct {
ID uint `uri:"id" json:"id" binding:"required"`
}
// Delete 删除节点
func (service *NodeService) Delete() serializer.Response {
// 查找用户组
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "节点不存在", err)
}
// 是否为系统节点
if node.ID <= 1 {
return serializer.Err(serializer.CodeNoPermissionErr, "系统节点无法删除", err)
}
cluster.Default.Delete(node.ID)
if err := model.DB.Delete(&node).Error; err != nil {
return serializer.DBErr("无法删除节点", err)
}
return serializer.Response{}
}
// Get 获取节点详情
func (service *NodeService) Get() serializer.Response {
node, err := model.GetNodeByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "节点不存在", err)
}
return serializer.Response{Data: node}
}

10
service/admin/policy.go
View File

@@ -151,7 +151,7 @@ func (service *PolicyService) AddCORS() serializer.Response {
case "oss":
handler := oss.Driver{
Policy: &policy,
HTTPClient: request.HTTPClient{},
HTTPClient: request.NewClient(),
}
if err := handler.CORS(); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "跨域策略添加失败", err)
@@ -161,7 +161,7 @@ func (service *PolicyService) AddCORS() serializer.Response {
b := &cossdk.BaseURL{BucketURL: u}
handler := cos.Driver{
Policy: &policy,
HTTPClient: request.HTTPClient{},
HTTPClient: request.NewClient(),
Client: cossdk.NewClient(b, &http.Client{
Transport: &cossdk.AuthorizationTransport{
SecretID: policy.AccessKey,
@@ -195,7 +195,7 @@ func (service *SlavePingService) Test() serializer.Response {
controller, _ := url.Parse("/api/v3/site/ping")
r := request.HTTPClient{}
r := request.NewClient()
res, err := r.Request(
"GET",
master.ResolveReference(controller).String(),
@@ -229,7 +229,7 @@ func (service *SlaveTestService) Test() serializer.Response {
}
bodyByte, _ := json.Marshal(body)
r := request.HTTPClient{}
r := request.NewClient()
res, err := r.Request(
"POST",
slave.ResolveReference(controller).String(),
@@ -245,7 +245,7 @@ func (service *SlaveTestService) Test() serializer.Response {
}
if res.Code != 0 {
return serializer.ParamErr("成功接到从机，但是"+res.Msg, nil)
return serializer.ParamErr("成功接到从机，但是从机返回："+res.Msg, nil)
}
return serializer.Response{}

57
service/aria2/add.go
View File

@@ -3,8 +3,14 @@ package aria2
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/monitor"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/slave"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
@@ -14,7 +20,7 @@ type AddURLService struct {
Dst string `json:"dst" binding:"required,min=1"`
}
// Add 创建新的链接离线下载任务
// Add 主机创建新的链接离线下载任务
func (service *AddURLService) Add(c *gin.Context, taskType int) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
@@ -35,19 +41,60 @@ func (service *AddURLService) Add(c *gin.Context, taskType int) serializer.Respo
// 创建任务
task := &model.Download{
Status: aria2.Ready,
Status: common.Ready,
Type: taskType,
Dst: service.Dst,
UserID: fs.User.ID,
Source: service.URL,
}
// 获取 Aria2 负载均衡器
aria2.Lock.RLock()
if err := aria2.Instance.CreateTask(task, fs.User.Group.OptionsSerialized.Aria2Options); err != nil {
aria2.Lock.RUnlock()
lb := aria2.LB
aria2.Lock.RUnlock()
// 获取 Aria2 实例
err, node := cluster.Default.BalanceNodeByFeature("aria2", lb)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Aria2 实例获取失败", err)
}
// 创建任务
gid, err := node.GetAria2Instance().CreateTask(task, fs.User.Group.OptionsSerialized.Aria2Options)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "任务创建失败", err)
}
aria2.Lock.RUnlock()
task.GID = gid
task.NodeID = node.ID()
_, err = task.Create()
if err != nil {
return serializer.DBErr("任务创建失败", err)
}
// 创建任务监控
monitor.NewMonitor(task)
return serializer.Response{}
}
// Add 从机创建新的链接离线下载任务
func Add(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 创建任务
gid, err := caller.(common.Aria2).CreateTask(service.Task, service.GroupOptions)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "无法创建离线下载任务", err)
}
// 创建事件通知回调
siteID, _ := c.Get("MasterSiteID")
mq.GlobalMQ.SubscribeCallback(gid, func(message mq.Message) {
if err := slave.DefaultController.SendNotification(siteID.(string), message.TriggeredBy, message); err != nil {
util.Log().Warning("无法发送离线下载任务状态变更通知, %s", err)
}
})
return serializer.Response{Data: gid}
}

81
service/aria2/manage.go
View File

@@ -2,7 +2,8 @@ package aria2
import (
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2/common"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
)
@@ -25,14 +26,14 @@ type DownloadListService struct {
// Finished 获取已完成的任务
func (service *DownloadListService) Finished(c *gin.Context, user *model.User) serializer.Response {
// 查找下载记录
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, aria2.Error, aria2.Complete, aria2.Canceled, aria2.Unknown)
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, common.Error, common.Complete, common.Canceled, common.Unknown)
return serializer.BuildFinishedListResponse(downloads)
}
// Downloading 获取正在下载中的任务
func (service *DownloadListService) Downloading(c *gin.Context, user *model.User) serializer.Response {
// 查找下载记录
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, aria2.Downloading, aria2.Paused, aria2.Ready)
downloads := model.GetDownloadsByStatusAndUser(service.Page, user.ID, common.Downloading, common.Paused, common.Ready)
return serializer.BuildDownloadingResponse(downloads)
}
@@ -47,7 +48,7 @@ func (service *DownloadTaskService) Delete(c *gin.Context) serializer.Response {
return serializer.Err(serializer.CodeNotFound, "下载记录不存在", err)
}
if download.Status >= aria2.Error {
if download.Status >= common.Error {
// 如果任务已完成，则删除任务记录
if err := download.Delete(); err != nil {
return serializer.Err(serializer.CodeDBError, "任务记录删除失败", err)
@@ -56,9 +57,12 @@ func (service *DownloadTaskService) Delete(c *gin.Context) serializer.Response {
}
// 取消任务
aria2.Lock.RLock()
defer aria2.Lock.RUnlock()
if err := aria2.Instance.Cancel(download); err != nil {
node := cluster.Default.GetNodeByID(download.GetNodeID())
if node == nil {
return serializer.Err(serializer.CodeInternalSetting, "目标节点不可用", err)
}
if err := node.GetAria2Instance().Cancel(download); err != nil {
return serializer.Err(serializer.CodeNotSet, "操作失败", err)
}
@@ -76,17 +80,72 @@ func (service *SelectFileService) Select(c *gin.Context) serializer.Response {
return serializer.Err(serializer.CodeNotFound, "下载记录不存在", err)
}
if download.StatusInfo.BitTorrent.Mode != "multi" || (download.Status != aria2.Downloading && download.Status != aria2.Paused) {
if download.StatusInfo.BitTorrent.Mode != "multi" || (download.Status != common.Downloading && download.Status != common.Paused) {
return serializer.Err(serializer.CodeNoPermissionErr, "此下载任务无法选取文件", err)
}
// 选取下载
aria2.Lock.RLock()
defer aria2.Lock.RUnlock()
if err := aria2.Instance.Select(download, service.Indexes); err != nil {
node := cluster.Default.GetNodeByID(download.GetNodeID())
if err := node.GetAria2Instance().Select(download, service.Indexes); err != nil {
return serializer.Err(serializer.CodeNotSet, "操作失败", err)
}
return serializer.Response{}
}
// SlaveStatus 从机查询离线任务状态
func SlaveStatus(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
status, err := caller.(common.Aria2).Status(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "离线下载任务查询失败", err)
}
return serializer.NewResponseWithGobData(status)
}
// SlaveCancel 取消从机离线下载任务
func SlaveCancel(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).Cancel(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "任务取消失败", err)
}
return serializer.Response{}
}
// SlaveSelect 从机选取离线下载任务文件
func SlaveSelect(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).Select(service.Task, service.Files)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "任务选取失败", err)
}
return serializer.Response{}
}
// SlaveSelect 从机选取离线下载任务文件
func SlaveDeleteTemp(c *gin.Context, service *serializer.SlaveAria2Call) serializer.Response {
caller, _ := c.Get("MasterAria2Instance")
// 查询任务
err := caller.(common.Aria2).DeleteTempFile(service.Task)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "临时文件删除失败", err)
}
return serializer.Response{}
}

128
service/explorer/file.go
View File

@@ -2,7 +2,6 @@ package explorer
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
@@ -20,7 +19,6 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
)
// SingleFileService 对单文件进行操作的五福，path为文件完整路径
@@ -43,29 +41,6 @@ type DownloadService struct {
ID string `uri:"id" binding:"required"`
}
// SlaveDownloadService 从机文件下載服务
type SlaveDownloadService struct {
PathEncoded string `uri:"path" binding:"required"`
Name string `uri:"name" binding:"required"`
Speed int `uri:"speed" binding:"min=0"`
}
// SlaveFileService 从机单文件文件相关服务
type SlaveFileService struct {
PathEncoded string `uri:"path" binding:"required"`
}
// SlaveFilesService 从机多文件相关服务
type SlaveFilesService struct {
Files []string `json:"files" binding:"required,gt=0"`
}
// SlaveListService 从机列表服务
type SlaveListService struct {
Path string `json:"path" binding:"required,min=1,max=65535"`
Recursive bool `json:"recursive"`
}
// New 创建新文件
func (service *SingleFileService) Create(c *gin.Context) serializer.Response {
// 创建文件系统
@@ -449,106 +424,3 @@ func (service *FileIDService) PutContent(ctx context.Context, c *gin.Context) se
Code: 0,
}
}
// ServeFile 通过签名的URL下载从机文件
func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Context, isDownload bool) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 解码文件路径
fileSource, err := base64.RawURLEncoding.DecodeString(service.PathEncoded)
if err != nil {
return serializer.ParamErr("无法解析的文件地址", err)
}
// 根据URL里的信息创建一个文件对象和用户对象
file := model.File{
Name: service.Name,
SourceName: string(fileSource),
Policy: model.Policy{
Model: gorm.Model{ID: 1},
Type: "local",
},
}
fs.User = &model.User{
Group: model.Group{SpeedLimit: service.Speed},
}
fs.FileTarget = []model.File{file}
// 开始处理下载
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
rs, err := fs.GetDownloadContent(ctx, 0)
if err != nil {
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}
defer rs.Close()
// 设置下载文件名
if isDownload {
c.Header("Content-Disposition", "attachment; filename=\""+url.PathEscape(fs.FileTarget[0].Name)+"\"")
}
// 发送文件
http.ServeContent(c.Writer, c.Request, fs.FileTarget[0].Name, time.Now(), rs)
return serializer.Response{
Code: 0,
}
}
// Delete 通过签名的URL删除从机文件
func (service *SlaveFilesService) Delete(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 删除文件
failed, err := fs.Handler.Delete(ctx, service.Files)
if err != nil {
// 将Data字段写为字符串方便主控端解析
data, _ := json.Marshal(serializer.RemoteDeleteRequest{Files: failed})
return serializer.Response{
Code: serializer.CodeNotFullySuccess,
Data: string(data),
Msg: fmt.Sprintf("有 %d 个文件未能成功删除", len(failed)),
Error: err.Error(),
}
}
return serializer.Response{Code: 0}
}
// Thumb 通过签名URL获取从机文件缩略图
func (service *SlaveFileService) Thumb(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 解码文件路径
fileSource, err := base64.RawURLEncoding.DecodeString(service.PathEncoded)
if err != nil {
return serializer.ParamErr("无法解析的文件地址", err)
}
fs.FileTarget = []model.File{{SourceName: string(fileSource), PicInfo: "1,1"}}
// 获取缩略图
resp, err := fs.GetThumb(ctx, 0)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "无法获取缩略图", err)
}
defer resp.Content.Close()
http.ServeContent(c.Writer, c.Request, "thumb.png", time.Now(), resp.Content)
return serializer.Response{Code: 0}
}

166
service/explorer/slave.go Normal file
View File

@@ -0,0 +1,166 @@
package explorer
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/slave"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
"github.com/cloudreve/Cloudreve/v3/pkg/task/slavetask"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"net/http"
"net/url"
"time"
)
// SlaveDownloadService 从机文件下載服务
type SlaveDownloadService struct {
PathEncoded string `uri:"path" binding:"required"`
Name string `uri:"name" binding:"required"`
Speed int `uri:"speed" binding:"min=0"`
}
// SlaveFileService 从机单文件文件相关服务
type SlaveFileService struct {
PathEncoded string `uri:"path" binding:"required"`
}
// SlaveFilesService 从机多文件相关服务
type SlaveFilesService struct {
Files []string `json:"files" binding:"required,gt=0"`
}
// SlaveListService 从机列表服务
type SlaveListService struct {
Path string `json:"path" binding:"required,min=1,max=65535"`
Recursive bool `json:"recursive"`
}
// ServeFile 通过签名的URL下载从机文件
func (service *SlaveDownloadService) ServeFile(ctx context.Context, c *gin.Context, isDownload bool) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 解码文件路径
fileSource, err := base64.RawURLEncoding.DecodeString(service.PathEncoded)
if err != nil {
return serializer.ParamErr("无法解析的文件地址", err)
}
// 根据URL里的信息创建一个文件对象和用户对象
file := model.File{
Name: service.Name,
SourceName: string(fileSource),
Policy: model.Policy{
Model: gorm.Model{ID: 1},
Type: "local",
},
}
fs.User = &model.User{
Group: model.Group{SpeedLimit: service.Speed},
}
fs.FileTarget = []model.File{file}
// 开始处理下载
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
rs, err := fs.GetDownloadContent(ctx, 0)
if err != nil {
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}
defer rs.Close()
// 设置下载文件名
if isDownload {
c.Header("Content-Disposition", "attachment; filename=\""+url.PathEscape(fs.FileTarget[0].Name)+"\"")
}
// 发送文件
http.ServeContent(c.Writer, c.Request, fs.FileTarget[0].Name, time.Now(), rs)
return serializer.Response{
Code: 0,
}
}
// Delete 通过签名的URL删除从机文件
func (service *SlaveFilesService) Delete(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 删除文件
failed, err := fs.Handler.Delete(ctx, service.Files)
if err != nil {
// 将Data字段写为字符串方便主控端解析
data, _ := json.Marshal(serializer.RemoteDeleteRequest{Files: failed})
return serializer.Response{
Code: serializer.CodeNotFullySuccess,
Data: string(data),
Msg: fmt.Sprintf("有 %d 个文件未能成功删除", len(failed)),
Error: err.Error(),
}
}
return serializer.Response{Code: 0}
}
// Thumb 通过签名URL获取从机文件缩略图
func (service *SlaveFileService) Thumb(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 解码文件路径
fileSource, err := base64.RawURLEncoding.DecodeString(service.PathEncoded)
if err != nil {
return serializer.ParamErr("无法解析的文件地址", err)
}
fs.FileTarget = []model.File{{SourceName: string(fileSource), PicInfo: "1,1"}}
// 获取缩略图
resp, err := fs.GetThumb(ctx, 0)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "无法获取缩略图", err)
}
defer resp.Content.Close()
http.ServeContent(c.Writer, c.Request, "thumb.png", time.Now(), resp.Content)
return serializer.Response{Code: 0}
}
// CreateTransferTask 创建从机文件转存任务
func CreateTransferTask(c *gin.Context, req *serializer.SlaveTransferReq) serializer.Response {
if id, ok := c.Get("MasterSiteID"); ok {
job := &slavetask.TransferTask{
Req: req,
MasterID: id.(string),
}
if err := slave.DefaultController.SubmitTask(job.MasterID, job, req.Hash(job.MasterID), func(job interface{}) {
task.TaskPoll.Submit(job.(task.Job))
}); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "任务创建失败", err)
}
return serializer.Response{}
}
return serializer.ParamErr("未知的主机节点ID", nil)
}

62
service/node/fabric.go Normal file
View File

@@ -0,0 +1,62 @@
package node
import (
"encoding/gob"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/onedrive"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/slave"
"github.com/gin-gonic/gin"
)
type SlaveNotificationService struct {
Subject string `uri:"subject" binding:"required"`
}
type OneDriveCredentialService struct {
PolicyID uint `uri:"id" binding:"required"`
}
func HandleMasterHeartbeat(req *serializer.NodePingReq) serializer.Response {
res, err := slave.DefaultController.HandleHeartBeat(req)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot initialize slave controller", err)
}
return serializer.Response{
Code: 0,
Data: res,
}
}
// HandleSlaveNotificationPush 转发从机的消息通知到本机消息队列
func (s *SlaveNotificationService) HandleSlaveNotificationPush(c *gin.Context) serializer.Response {
var msg mq.Message
dec := gob.NewDecoder(c.Request.Body)
if err := dec.Decode(&msg); err != nil {
return serializer.ParamErr("Cannot parse notification message", err)
}
mq.GlobalMQ.Publish(s.Subject, msg)
return serializer.Response{}
}
// Get 获取主机OneDrive策略的AccessToken
func (s *OneDriveCredentialService) Get(c *gin.Context) serializer.Response {
policy, err := model.GetPolicyByID(s.PolicyID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "Cannot found storage policy", err)
}
client, err := onedrive.NewClient(&policy)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot initialize OneDrive client", err)
}
if err := client.UpdateCredential(c); err != nil {
return serializer.Err(serializer.CodeInternalSetting, "Cannot refresh OneDrive credential", err)
}
return serializer.Response{Data: client.Credential.AccessToken}
}