mirror of
https://github.com/bufanyun/hotgo.git
synced 2025-08-28 05:12:32 +08:00
发布v2.13.1版本,更新内容请查看:https://github.com/bufanyun/hotgo/blob/v2.0/docs/guide-zh-CN/start-update-log.md
This commit is contained in:
@@ -5,13 +5,65 @@
|
||||
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
|
||||
package storager
|
||||
|
||||
import (
|
||||
"github.com/gogf/gf/v2/net/ghttp"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
"hotgo/internal/model/entity"
|
||||
)
|
||||
|
||||
// FileMeta 文件元数据
|
||||
type FileMeta struct {
|
||||
Filename string // 文件名称
|
||||
Size int64 // 文件大小
|
||||
Kind string // 文件上传类型
|
||||
MimeType string // 文件扩展类型
|
||||
NaiveType string // NaiveUI类型
|
||||
Ext string // 文件扩展名
|
||||
Md5 string // 文件hash
|
||||
Filename string `json:"filename"` // 文件名称
|
||||
Size int64 `json:"size"` // 文件大小
|
||||
Kind string `json:"kind"` // 文件上传类型
|
||||
MimeType string `json:"mimeType"` // 文件扩展类型
|
||||
NaiveType string `json:"naiveType"` // NaiveUI类型
|
||||
Ext string `json:"ext"` // 文件扩展名
|
||||
Md5 string `json:"md5"` // 文件hash
|
||||
}
|
||||
|
||||
// MultipartProgress 分片进度
|
||||
type MultipartProgress struct {
|
||||
UploadId string `json:"uploadId"` // 上传事件ID
|
||||
ThirdUploadId string `json:"thirdUploadId"` // 第三方上传事件ID
|
||||
Meta *FileMeta `json:"meta"` // 文件元数据
|
||||
ShardCount int `json:"shardCount"` // 分片数量
|
||||
UploadedIndex []int `json:"uploadedIndex"` // 已上传的分片索引
|
||||
CreatedAt *gtime.Time `json:"createdAt"` // 创建时间
|
||||
}
|
||||
|
||||
// CheckMultipartParams 检查文件分片
|
||||
type CheckMultipartParams struct {
|
||||
UploadType string `json:"uploadType" dc:"文件类型"`
|
||||
FileName string `json:"fileName" dc:"文件名称"`
|
||||
Size int64 `json:"size" dc:"文件大小"`
|
||||
Md5 string `json:"md5" dc:"文件md5值"`
|
||||
ShardCount int `json:"shardCount" dc:"分片数量"`
|
||||
meta *FileMeta
|
||||
}
|
||||
|
||||
type CheckMultipartModel struct {
|
||||
UploadId string `json:"uploadId" dc:"上传事件ID"`
|
||||
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
|
||||
WaitUploadIndex []int `json:"waitUploadIndex" dc:"等待上传的分片索引"`
|
||||
Progress float64 `json:"progress" dc:"上传进度"`
|
||||
SizeFormat string `json:"sizeFormat" dc:"文件大小"`
|
||||
}
|
||||
|
||||
// UploadPartParams 分片上传
|
||||
type UploadPartParams struct {
|
||||
UploadId string `json:"uploadId" dc:"上传事件ID"`
|
||||
UploadType string `json:"uploadType" dc:"文件类型"`
|
||||
FileName string `json:"fileName" dc:"文件名称"`
|
||||
Size int64 `json:"size" dc:"文件大小"`
|
||||
Md5 string `json:"md5" dc:"文件md5值"`
|
||||
Index int `json:"index" dc:"分片索引"`
|
||||
File *ghttp.UploadFile `json:"file" type:"file" dc:"分片文件"`
|
||||
mp *MultipartProgress
|
||||
}
|
||||
|
||||
type UploadPartModel struct {
|
||||
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
|
||||
Progress float64 `json:"progress" dc:"上传进度"`
|
||||
Finish bool `json:"finish" dc:"是否完成"`
|
||||
}
|
||||
|
@@ -14,18 +14,26 @@ import (
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
"github.com/gogf/gf/v2/util/grand"
|
||||
"hotgo/internal/consts"
|
||||
"hotgo/internal/library/cache"
|
||||
"hotgo/internal/library/contexts"
|
||||
"hotgo/internal/model/entity"
|
||||
"hotgo/utility/convert"
|
||||
"hotgo/utility/format"
|
||||
"hotgo/utility/url"
|
||||
"hotgo/utility/validate"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// UploadDrive 存储驱动
|
||||
type UploadDrive interface {
|
||||
// Upload 上传
|
||||
Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error)
|
||||
// CreateMultipart 创建分片事件
|
||||
CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error)
|
||||
// UploadPart 上传分片
|
||||
UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error)
|
||||
}
|
||||
|
||||
// New 初始化存储驱动
|
||||
@@ -70,6 +78,31 @@ func DoUpload(ctx context.Context, typ string, file *ghttp.UploadFile) (result *
|
||||
return
|
||||
}
|
||||
|
||||
if err = ValidateFileMeta(typ, meta); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
result, err = HasFile(ctx, meta.Md5)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 相同存储相同身份才复用
|
||||
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
// 上传到驱动
|
||||
fullPath, err := New(config.Drive).Upload(ctx, file)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// 写入附件记录
|
||||
return write(ctx, meta, fullPath)
|
||||
}
|
||||
|
||||
// ValidateFileMeta 验证文件元数据
|
||||
func ValidateFileMeta(typ string, meta *FileMeta) (err error) {
|
||||
if _, err = GetFileMimeType(meta.Ext); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -123,24 +156,7 @@ func DoUpload(ctx context.Context, typ string, file *ghttp.UploadFile) (result *
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
result, err = hasFile(ctx, meta.Md5)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 相同存储相同身份才复用
|
||||
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
// 上传到驱动
|
||||
fullPath, err := New(config.Drive).Upload(ctx, file)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// 写入附件记录
|
||||
return write(ctx, meta, fullPath)
|
||||
return
|
||||
}
|
||||
|
||||
// LastUrl 根据驱动获取最终文件访问地址
|
||||
@@ -225,8 +241,8 @@ func write(ctx context.Context, meta *FileMeta, fullPath string) (models *entity
|
||||
return
|
||||
}
|
||||
|
||||
// hasFile 检查附件是否存在
|
||||
func hasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
|
||||
// HasFile 检查附件是否存在
|
||||
func HasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
|
||||
if err = GetModel(ctx).Where("md5", md5).Scan(&res); err != nil {
|
||||
err = gerror.Wrap(err, "检查文件hash时出现错误")
|
||||
return
|
||||
@@ -238,10 +254,147 @@ func hasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err er
|
||||
|
||||
// 只有在上传时才会检查md5值,如果附件存在则更新最后上传时间,保证上传列表更新显示在最前面
|
||||
if res.Id > 0 {
|
||||
_, _ = GetModel(ctx).WherePri(res.Id).Data(g.Map{
|
||||
update := g.Map{
|
||||
"status": consts.StatusEnabled,
|
||||
"updated_at": gtime.Now(),
|
||||
}).Update()
|
||||
}
|
||||
_, _ = GetModel(ctx).WherePri(res.Id).Data(update).Update()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// CheckMultipart 检查文件分片
|
||||
func CheckMultipart(ctx context.Context, in *CheckMultipartParams) (res *CheckMultipartModel, err error) {
|
||||
res = new(CheckMultipartModel)
|
||||
|
||||
meta := new(FileMeta)
|
||||
meta.Filename = in.FileName
|
||||
meta.Size = in.Size
|
||||
meta.Ext = Ext(in.FileName)
|
||||
meta.Kind = GetFileKind(meta.Ext)
|
||||
meta.MimeType, err = GetFileMimeType(meta.Ext)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 兼容naiveUI
|
||||
naiveType := "text/plain"
|
||||
if IsImgType(Ext(in.FileName)) {
|
||||
naiveType = ""
|
||||
}
|
||||
meta.NaiveType = naiveType
|
||||
meta.Md5 = in.Md5
|
||||
|
||||
if err = ValidateFileMeta(in.UploadType, meta); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
result, err := HasFile(ctx, in.Md5)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 文件已存在,直接返回。相同存储相同身份才复用
|
||||
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
|
||||
res.Attachment = result
|
||||
return
|
||||
}
|
||||
|
||||
for i := 0; i < in.ShardCount; i++ {
|
||||
res.WaitUploadIndex = append(res.WaitUploadIndex, i+1)
|
||||
}
|
||||
|
||||
in.meta = meta
|
||||
progress, err := GetOrCreateMultipartProgress(ctx, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(progress.UploadedIndex) > 0 {
|
||||
res.WaitUploadIndex = convert.DifferenceSlice(progress.UploadedIndex, res.WaitUploadIndex)
|
||||
}
|
||||
|
||||
if len(res.WaitUploadIndex) == 0 {
|
||||
res.WaitUploadIndex = make([]int, 0)
|
||||
}
|
||||
res.UploadId = progress.UploadId
|
||||
res.Progress = CalcUploadProgress(progress.UploadedIndex, progress.ShardCount)
|
||||
res.SizeFormat = format.FileSize(progress.Meta.Size)
|
||||
return
|
||||
}
|
||||
|
||||
// CalcUploadProgress 计算上传进度
|
||||
func CalcUploadProgress(uploadedIndex []int, shardCount int) float64 {
|
||||
return format.Round2Float64(float64(len(uploadedIndex)) / float64(shardCount) * 100)
|
||||
}
|
||||
|
||||
// GenUploadId 生成上传ID
|
||||
func GenUploadId(ctx context.Context, md5 string) string {
|
||||
return fmt.Sprintf("%v:%v:%v@%v", md5, contexts.GetUserId(ctx), contexts.GetModule(ctx), config.Drive)
|
||||
}
|
||||
|
||||
// GetOrCreateMultipartProgress 获取或创建分片上传事件进度
|
||||
func GetOrCreateMultipartProgress(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
uploadId := GenUploadId(ctx, in.Md5)
|
||||
res, err = GetMultipartProgress(ctx, uploadId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res != nil {
|
||||
return res, nil
|
||||
}
|
||||
return New(config.Drive).CreateMultipart(ctx, in)
|
||||
}
|
||||
|
||||
// GetMultipartProgress 获取分片上传事件进度
|
||||
func GetMultipartProgress(ctx context.Context, uploadId string) (res *MultipartProgress, err error) {
|
||||
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, uploadId)
|
||||
get, err := cache.Instance().Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = get.Scan(&res)
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipartProgress 创建分片上传事件进度
|
||||
func CreateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
|
||||
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
|
||||
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
|
||||
}
|
||||
|
||||
// UpdateMultipartProgress 更新分片上传事件进度
|
||||
func UpdateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
|
||||
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
|
||||
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
|
||||
}
|
||||
|
||||
// DelMultipartProgress 删除分片上传事件进度
|
||||
func DelMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
|
||||
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
|
||||
_, err = cache.Instance().Remove(ctx, key)
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
in.mp, err = GetMultipartProgress(ctx, in.UploadId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.mp == nil {
|
||||
err = gerror.New("分片事件不存在,请重新上传!")
|
||||
return
|
||||
}
|
||||
|
||||
if validate.InSlice(in.mp.UploadedIndex, in.Index) {
|
||||
err = gerror.New("该分片已上传过了")
|
||||
return
|
||||
}
|
||||
|
||||
res, err = New(config.Drive).UploadPart(ctx, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
@@ -45,3 +45,15 @@ func (d *CosDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath
|
||||
_, err = client.Object.Put(ctx, fullPath, f2, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *CosDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *CosDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
@@ -10,7 +10,11 @@ import (
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/net/ghttp"
|
||||
"github.com/gogf/gf/v2/os/gfile"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
"github.com/gogf/gf/v2/util/gconv"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -45,3 +49,102 @@ func (d *LocalDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
|
||||
fullPath = config.LocalPath + nowDate + "/" + fileName
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *LocalDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (mp *MultipartProgress, err error) {
|
||||
mp = new(MultipartProgress)
|
||||
mp.UploadId = GenUploadId(ctx, in.Md5)
|
||||
mp.Meta = in.meta
|
||||
mp.ShardCount = in.ShardCount
|
||||
mp.UploadedIndex = make([]int, 0)
|
||||
mp.CreatedAt = gtime.Now()
|
||||
if err = CreateMultipartProgress(ctx, mp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *LocalDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
sp := g.Cfg().MustGet(ctx, "server.serverRoot")
|
||||
if sp.IsEmpty() {
|
||||
err = gerror.New("本地上传驱动必须配置静态路径!")
|
||||
return
|
||||
}
|
||||
|
||||
spStr := strings.Trim(sp.String(), "/") + "/"
|
||||
|
||||
if config.LocalPath == "" {
|
||||
err = gerror.New("本地上传驱动必须配置本地存储路径!")
|
||||
return
|
||||
}
|
||||
|
||||
// 分片文件存放路径
|
||||
partFilePath := spStr + config.LocalPath + "tmp/" + in.Md5
|
||||
|
||||
// 写入文件
|
||||
in.File.Filename = gconv.String(in.Index)
|
||||
if _, err = in.File.Save(partFilePath, false); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 更新上传进度
|
||||
in.mp.UploadedIndex = append(in.mp.UploadedIndex, in.Index)
|
||||
if err = UpdateMultipartProgress(ctx, in.mp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res = new(UploadPartModel)
|
||||
|
||||
// 已全部上传完毕
|
||||
if len(in.mp.UploadedIndex) == in.mp.ShardCount {
|
||||
// 删除进度统计
|
||||
if err = DelMultipartProgress(ctx, in.mp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 合并文件
|
||||
finalDirPath := GenFullPath(config.LocalPath, gfile.Ext(in.mp.Meta.Filename))
|
||||
if err = MergePartFile(partFilePath, spStr+finalDirPath); err != nil {
|
||||
err = gerror.Newf("合并分片文件出错:%v", err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 删除临时分片
|
||||
if err = os.RemoveAll(partFilePath); err != nil {
|
||||
err = gerror.Newf("删除临时分片文件出错:%v", err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 写入附件记录
|
||||
attachment, err := write(ctx, in.mp.Meta, finalDirPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res.Finish = true
|
||||
res.Progress = 100
|
||||
res.Attachment = attachment
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// 计算上传进度
|
||||
res.Progress = CalcUploadProgress(in.mp.UploadedIndex, in.mp.ShardCount)
|
||||
return
|
||||
}
|
||||
|
||||
// MergePartFile 合并分片文件
|
||||
func MergePartFile(srcPath, dstPath string) (err error) {
|
||||
dir, err := os.ReadDir(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, file := range dir {
|
||||
filePath := filepath.Join(srcPath, file.Name())
|
||||
if err = gfile.PutBytesAppend(dstPath, gfile.GetBytes(filePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
@@ -62,3 +62,15 @@ func (d *MinioDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
|
||||
_, err = client.PutObject(ctx, config.MinioBucket, fullPath, reader, file.Size, opts)
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *MinioDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *MinioDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
@@ -45,3 +45,15 @@ func (d *OssDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath
|
||||
err = bucket.PutObject(fullPath, f2)
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *OssDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *OssDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
@@ -55,3 +55,15 @@ func (d *QiNiuDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
|
||||
err = storage.NewFormUploader(&cfg).Put(ctx, &storage.PutRet{}, token, fullPath, f2, file.Size, &storage.PutExtra{})
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *QiNiuDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *QiNiuDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
@@ -48,3 +48,15 @@ func (d *UCloudDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullP
|
||||
err = client.IOPut(f2, fullPath, "")
|
||||
return
|
||||
}
|
||||
|
||||
// CreateMultipart 创建分片事件
|
||||
func (d *UCloudDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
||||
// UploadPart 上传分片
|
||||
func (d *UCloudDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
|
||||
err = gerror.New("当前驱动暂不支持分片上传!")
|
||||
return
|
||||
}
|
||||
|
Reference in New Issue
Block a user