This commit is contained in:
孟帅
2024-03-07 20:08:56 +08:00
parent 6dd8cbadad
commit 0fbc1ad47c
246 changed files with 9441 additions and 2293 deletions

View File

@@ -5,13 +5,65 @@
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"hotgo/internal/model/entity"
)
// FileMeta 文件元数据
type FileMeta struct {
Filename string // 文件名称
Size int64 // 文件大小
Kind string // 文件上传类型
MimeType string // 文件扩展类型
NaiveType string // NaiveUI类型
Ext string // 文件扩展名
Md5 string // 文件hash
Filename string `json:"filename"` // 文件名称
Size int64 `json:"size"` // 文件大小
Kind string `json:"kind"` // 文件上传类型
MimeType string `json:"mimeType"` // 文件扩展类型
NaiveType string `json:"naiveType"` // NaiveUI类型
Ext string `json:"ext"` // 文件扩展名
Md5 string `json:"md5"` // 文件hash
}
// MultipartProgress 分片进度
type MultipartProgress struct {
UploadId string `json:"uploadId"` // 上传事件ID
ThirdUploadId string `json:"thirdUploadId"` // 第三方上传事件ID
Meta *FileMeta `json:"meta"` // 文件元数据
ShardCount int `json:"shardCount"` // 分片数量
UploadedIndex []int `json:"uploadedIndex"` // 已上传的分片索引
CreatedAt *gtime.Time `json:"createdAt"` // 创建时间
}
// CheckMultipartParams 检查文件分片
type CheckMultipartParams struct {
UploadType string `json:"uploadType" dc:"文件类型"`
FileName string `json:"fileName" dc:"文件名称"`
Size int64 `json:"size" dc:"文件大小"`
Md5 string `json:"md5" dc:"文件md5值"`
ShardCount int `json:"shardCount" dc:"分片数量"`
meta *FileMeta
}
type CheckMultipartModel struct {
UploadId string `json:"uploadId" dc:"上传事件ID"`
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
WaitUploadIndex []int `json:"waitUploadIndex" dc:"等待上传的分片索引"`
Progress float64 `json:"progress" dc:"上传进度"`
SizeFormat string `json:"sizeFormat" dc:"文件大小"`
}
// UploadPartParams 分片上传
type UploadPartParams struct {
UploadId string `json:"uploadId" dc:"上传事件ID"`
UploadType string `json:"uploadType" dc:"文件类型"`
FileName string `json:"fileName" dc:"文件名称"`
Size int64 `json:"size" dc:"文件大小"`
Md5 string `json:"md5" dc:"文件md5值"`
Index int `json:"index" dc:"分片索引"`
File *ghttp.UploadFile `json:"file" type:"file" dc:"分片文件"`
mp *MultipartProgress
}
type UploadPartModel struct {
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
Progress float64 `json:"progress" dc:"上传进度"`
Finish bool `json:"finish" dc:"是否完成"`
}

View File

@@ -14,18 +14,26 @@ import (
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/grand"
"hotgo/internal/consts"
"hotgo/internal/library/cache"
"hotgo/internal/library/contexts"
"hotgo/internal/model/entity"
"hotgo/utility/convert"
"hotgo/utility/format"
"hotgo/utility/url"
"hotgo/utility/validate"
"strconv"
"strings"
"time"
)
// UploadDrive 存储驱动
type UploadDrive interface {
// Upload 上传
Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error)
// CreateMultipart 创建分片事件
CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error)
// UploadPart 上传分片
UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error)
}
// New 初始化存储驱动
@@ -70,6 +78,31 @@ func DoUpload(ctx context.Context, typ string, file *ghttp.UploadFile) (result *
return
}
if err = ValidateFileMeta(typ, meta); err != nil {
return
}
result, err = HasFile(ctx, meta.Md5)
if err != nil {
return
}
// 相同存储相同身份才复用
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
return
}
// 上传到驱动
fullPath, err := New(config.Drive).Upload(ctx, file)
if err != nil {
return
}
// 写入附件记录
return write(ctx, meta, fullPath)
}
// ValidateFileMeta 验证文件元数据
func ValidateFileMeta(typ string, meta *FileMeta) (err error) {
if _, err = GetFileMimeType(meta.Ext); err != nil {
return
}
@@ -123,24 +156,7 @@ func DoUpload(ctx context.Context, typ string, file *ghttp.UploadFile) (result *
return
}
}
result, err = hasFile(ctx, meta.Md5)
if err != nil {
return
}
// 相同存储相同身份才复用
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
return
}
// 上传到驱动
fullPath, err := New(config.Drive).Upload(ctx, file)
if err != nil {
return
}
// 写入附件记录
return write(ctx, meta, fullPath)
return
}
// LastUrl 根据驱动获取最终文件访问地址
@@ -225,8 +241,8 @@ func write(ctx context.Context, meta *FileMeta, fullPath string) (models *entity
return
}
// hasFile 检查附件是否存在
func hasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
// HasFile 检查附件是否存在
func HasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
if err = GetModel(ctx).Where("md5", md5).Scan(&res); err != nil {
err = gerror.Wrap(err, "检查文件hash时出现错误")
return
@@ -238,10 +254,147 @@ func hasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err er
// 只有在上传时才会检查md5值如果附件存在则更新最后上传时间保证上传列表更新显示在最前面
if res.Id > 0 {
_, _ = GetModel(ctx).WherePri(res.Id).Data(g.Map{
update := g.Map{
"status": consts.StatusEnabled,
"updated_at": gtime.Now(),
}).Update()
}
_, _ = GetModel(ctx).WherePri(res.Id).Data(update).Update()
}
return
}
// CheckMultipart 检查文件分片
func CheckMultipart(ctx context.Context, in *CheckMultipartParams) (res *CheckMultipartModel, err error) {
res = new(CheckMultipartModel)
meta := new(FileMeta)
meta.Filename = in.FileName
meta.Size = in.Size
meta.Ext = Ext(in.FileName)
meta.Kind = GetFileKind(meta.Ext)
meta.MimeType, err = GetFileMimeType(meta.Ext)
if err != nil {
return
}
// 兼容naiveUI
naiveType := "text/plain"
if IsImgType(Ext(in.FileName)) {
naiveType = ""
}
meta.NaiveType = naiveType
meta.Md5 = in.Md5
if err = ValidateFileMeta(in.UploadType, meta); err != nil {
return
}
result, err := HasFile(ctx, in.Md5)
if err != nil {
return nil, err
}
// 文件已存在,直接返回。相同存储相同身份才复用
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
res.Attachment = result
return
}
for i := 0; i < in.ShardCount; i++ {
res.WaitUploadIndex = append(res.WaitUploadIndex, i+1)
}
in.meta = meta
progress, err := GetOrCreateMultipartProgress(ctx, in)
if err != nil {
return nil, err
}
if len(progress.UploadedIndex) > 0 {
res.WaitUploadIndex = convert.DifferenceSlice(progress.UploadedIndex, res.WaitUploadIndex)
}
if len(res.WaitUploadIndex) == 0 {
res.WaitUploadIndex = make([]int, 0)
}
res.UploadId = progress.UploadId
res.Progress = CalcUploadProgress(progress.UploadedIndex, progress.ShardCount)
res.SizeFormat = format.FileSize(progress.Meta.Size)
return
}
// CalcUploadProgress 计算上传进度
func CalcUploadProgress(uploadedIndex []int, shardCount int) float64 {
return format.Round2Float64(float64(len(uploadedIndex)) / float64(shardCount) * 100)
}
// GenUploadId 生成上传ID
func GenUploadId(ctx context.Context, md5 string) string {
return fmt.Sprintf("%v:%v:%v@%v", md5, contexts.GetUserId(ctx), contexts.GetModule(ctx), config.Drive)
}
// GetOrCreateMultipartProgress 获取或创建分片上传事件进度
func GetOrCreateMultipartProgress(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
uploadId := GenUploadId(ctx, in.Md5)
res, err = GetMultipartProgress(ctx, uploadId)
if err != nil {
return nil, err
}
if res != nil {
return res, nil
}
return New(config.Drive).CreateMultipart(ctx, in)
}
// GetMultipartProgress 获取分片上传事件进度
func GetMultipartProgress(ctx context.Context, uploadId string) (res *MultipartProgress, err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, uploadId)
get, err := cache.Instance().Get(ctx, key)
if err != nil {
return nil, err
}
err = get.Scan(&res)
return
}
// CreateMultipartProgress 创建分片上传事件进度
func CreateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
}
// UpdateMultipartProgress 更新分片上传事件进度
func UpdateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
}
// DelMultipartProgress 删除分片上传事件进度
func DelMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
_, err = cache.Instance().Remove(ctx, key)
return
}
// UploadPart 上传分片
func UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
in.mp, err = GetMultipartProgress(ctx, in.UploadId)
if err != nil {
return nil, err
}
if in.mp == nil {
err = gerror.New("分片事件不存在,请重新上传!")
return
}
if validate.InSlice(in.mp.UploadedIndex, in.Index) {
err = gerror.New("该分片已上传过了")
return
}
res, err = New(config.Drive).UploadPart(ctx, in)
if err != nil {
return nil, err
}
return
}

View File

@@ -45,3 +45,15 @@ func (d *CosDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath
_, err = client.Object.Put(ctx, fullPath, f2, nil)
return
}
// CreateMultipart 创建分片事件
func (d *CosDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}
// UploadPart 上传分片
func (d *CosDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}

View File

@@ -10,7 +10,11 @@ import (
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"os"
"path/filepath"
"strings"
)
@@ -45,3 +49,102 @@ func (d *LocalDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
fullPath = config.LocalPath + nowDate + "/" + fileName
return
}
// CreateMultipart 创建分片事件
func (d *LocalDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (mp *MultipartProgress, err error) {
mp = new(MultipartProgress)
mp.UploadId = GenUploadId(ctx, in.Md5)
mp.Meta = in.meta
mp.ShardCount = in.ShardCount
mp.UploadedIndex = make([]int, 0)
mp.CreatedAt = gtime.Now()
if err = CreateMultipartProgress(ctx, mp); err != nil {
return nil, err
}
return
}
// UploadPart 上传分片
func (d *LocalDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
sp := g.Cfg().MustGet(ctx, "server.serverRoot")
if sp.IsEmpty() {
err = gerror.New("本地上传驱动必须配置静态路径!")
return
}
spStr := strings.Trim(sp.String(), "/") + "/"
if config.LocalPath == "" {
err = gerror.New("本地上传驱动必须配置本地存储路径!")
return
}
// 分片文件存放路径
partFilePath := spStr + config.LocalPath + "tmp/" + in.Md5
// 写入文件
in.File.Filename = gconv.String(in.Index)
if _, err = in.File.Save(partFilePath, false); err != nil {
return
}
// 更新上传进度
in.mp.UploadedIndex = append(in.mp.UploadedIndex, in.Index)
if err = UpdateMultipartProgress(ctx, in.mp); err != nil {
return nil, err
}
res = new(UploadPartModel)
// 已全部上传完毕
if len(in.mp.UploadedIndex) == in.mp.ShardCount {
// 删除进度统计
if err = DelMultipartProgress(ctx, in.mp); err != nil {
return nil, err
}
// 合并文件
finalDirPath := GenFullPath(config.LocalPath, gfile.Ext(in.mp.Meta.Filename))
if err = MergePartFile(partFilePath, spStr+finalDirPath); err != nil {
err = gerror.Newf("合并分片文件出错:%v", err.Error())
return nil, err
}
// 删除临时分片
if err = os.RemoveAll(partFilePath); err != nil {
err = gerror.Newf("删除临时分片文件出错:%v", err.Error())
return nil, err
}
// 写入附件记录
attachment, err := write(ctx, in.mp.Meta, finalDirPath)
if err != nil {
return nil, err
}
res.Finish = true
res.Progress = 100
res.Attachment = attachment
return res, nil
}
// 计算上传进度
res.Progress = CalcUploadProgress(in.mp.UploadedIndex, in.mp.ShardCount)
return
}
// MergePartFile 合并分片文件
func MergePartFile(srcPath, dstPath string) (err error) {
dir, err := os.ReadDir(srcPath)
if err != nil {
return err
}
for _, file := range dir {
filePath := filepath.Join(srcPath, file.Name())
if err = gfile.PutBytesAppend(dstPath, gfile.GetBytes(filePath)); err != nil {
return err
}
}
return
}

View File

@@ -62,3 +62,15 @@ func (d *MinioDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
_, err = client.PutObject(ctx, config.MinioBucket, fullPath, reader, file.Size, opts)
return
}
// CreateMultipart 创建分片事件
func (d *MinioDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}
// UploadPart 上传分片
func (d *MinioDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}

View File

@@ -45,3 +45,15 @@ func (d *OssDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath
err = bucket.PutObject(fullPath, f2)
return
}
// CreateMultipart 创建分片事件
func (d *OssDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}
// UploadPart 上传分片
func (d *OssDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}

View File

@@ -55,3 +55,15 @@ func (d *QiNiuDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPa
err = storage.NewFormUploader(&cfg).Put(ctx, &storage.PutRet{}, token, fullPath, f2, file.Size, &storage.PutExtra{})
return
}
// CreateMultipart 创建分片事件
func (d *QiNiuDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}
// UploadPart 上传分片
func (d *QiNiuDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}

View File

@@ -48,3 +48,15 @@ func (d *UCloudDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullP
err = client.IOPut(f2, fullPath, "")
return
}
// CreateMultipart 创建分片事件
func (d *UCloudDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}
// UploadPart 上传分片
func (d *UCloudDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传!")
return
}