模块化上传驱动,使用泛型优化工具库降低冗余

This commit is contained in:
孟帅
2023-06-02 20:29:08 +08:00
parent fdc48b9335
commit 62ecbb7f26
96 changed files with 1276 additions and 1483 deletions

View File

@@ -23,10 +23,8 @@ func Tpl(name, tpl string) string {
// 最终效果:/应用名称/插件模块名称/xxx/xxx。如果你不喜欢现在的路由风格可以自行调整
func RouterPrefix(ctx context.Context, app, name string) string {
var prefix = "/"
if app != "" {
prefix = g.Cfg().MustGet(ctx, "router."+app+".prefix", "/"+app+"").String()
}
return prefix + "/" + name
}

View File

@@ -91,7 +91,6 @@ func Build(ctx context.Context, sk Skeleton, conf *model.BuildAddonConfig) (err
if err = gfile.PutContents(webViewsPath+"/config/system.vue", gstr.ReplaceByMap(webConfigSystem, replaces)); err != nil {
return
}
return
}

View File

@@ -141,6 +141,5 @@ func ModuleSelect() form.Selects {
Name: skeleton.Label,
})
}
return lst
}

View File

@@ -104,21 +104,18 @@ func (a *adapter) model() *gdb.Model {
// create a policy table when it's not exists.
func (a *adapter) createPolicyTable() (err error) {
_, err = a.db.Exec(context.TODO(), fmt.Sprintf(createPolicyTableSql, a.table))
return
}
// drop policy table from the storage.
func (a *adapter) dropPolicyTable() (err error) {
_, err = a.db.Exec(context.TODO(), fmt.Sprintf(dropPolicyTableSql, a.table))
return
}
// LoadPolicy loads all policy rules from the storage.
func (a *adapter) LoadPolicy(model model.Model) (err error) {
var rules []policyRule
if err = a.model().Scan(&rules); err != nil {
return
}
@@ -126,7 +123,6 @@ func (a *adapter) LoadPolicy(model model.Model) (err error) {
for _, rule := range rules {
a.loadPolicyRule(rule, model)
}
return
}
@@ -159,14 +155,12 @@ func (a *adapter) SavePolicy(model model.Model) (err error) {
return
}
}
return
}
// AddPolicy adds a policy rule to the storage.
func (a *adapter) AddPolicy(sec string, ptype string, rule []string) (err error) {
_, err = a.model().Insert(a.buildPolicyRule(ptype, rule))
return
}
@@ -183,7 +177,6 @@ func (a *adapter) AddPolicies(sec string, ptype string, rules [][]string) (err e
}
_, err = a.model().Insert(policyRules)
return
}
@@ -228,14 +221,12 @@ func (a *adapter) RemovePolicies(sec string, ptype string, rules [][]string) (er
}
_, err = db.Delete()
return
}
// UpdatePolicy updates a policy rule from storage.
func (a *adapter) UpdatePolicy(sec string, ptype string, oldRule, newRule []string) (err error) {
_, err = a.model().Update(a.buildPolicyRule(ptype, newRule), a.buildPolicyRule(ptype, oldRule))
return
}
@@ -244,18 +235,14 @@ func (a *adapter) UpdatePolicies(sec string, ptype string, oldRules, newRules []
if len(oldRules) == 0 || len(newRules) == 0 {
return
}
err = a.db.Transaction(context.TODO(), func(ctx context.Context, tx gdb.TX) error {
return a.db.Transaction(context.TODO(), func(ctx context.Context, tx gdb.TX) error {
for i := 0; i < int(math.Min(float64(len(oldRules)), float64(len(newRules)))); i++ {
if _, err = tx.Model(a.table).Update(a.buildPolicyRule(ptype, newRules[i]), a.buildPolicyRule(ptype, oldRules[i])); err != nil {
return err
}
}
return nil
})
return
}
// 加载策略规则
@@ -285,7 +272,6 @@ func (a *adapter) loadPolicyRule(rule policyRule, model model.Model) {
if rule.V5 != "" {
ruleText += ", " + rule.V5
}
if err := persist.LoadPolicyLine(ruleText, model); err != nil {
panic(err)
}
@@ -318,6 +304,5 @@ func (a *adapter) buildPolicyRule(ptype string, data []string) policyRule {
if len(data) > 5 {
rule.V5 = data[5]
}
return rule
}

View File

@@ -66,7 +66,6 @@ func GetUser(ctx context.Context) *model.Identity {
if c == nil {
return nil
}
return c.User
}

View File

@@ -3,7 +3,6 @@
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
//
package ems
import (
@@ -47,6 +46,5 @@ func sendToMail(config *model.EmailConfig, to, subject, body, mailType string) e
}
msg := []byte("To: " + to + "\r\nFrom: " + config.SendName + "<" + config.User + ">" + "\r\nSubject: " + subject + "\r\n" + contentType + "\r\n\r\n" + body)
return smtp.SendMail(config.Addr, auth, config.User, sendTo, msg)
}

View File

@@ -33,7 +33,6 @@ func Dao(ctx context.Context) (err error) {
}
gendao.DoGenDaoForArray(ctx, inp)
}
return
}
@@ -126,7 +125,6 @@ func TableSelects(ctx context.Context, in sysin.GenCodesSelectsInp) (res *sysin.
}
res.Addons = addons.ModuleSelect()
return
}
@@ -163,7 +161,6 @@ func GenTypeSelect(ctx context.Context) (res sysin.GenTypeSelects, err error) {
res = append(res, row)
}
sort.Sort(res)
return
}

View File

@@ -3,7 +3,6 @@
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
//
package hggen
import (
@@ -77,7 +76,6 @@ func GetDaoConfig(group string) gendao.CGenDaoInput {
panic(err)
}
}
return inp
}

View File

@@ -81,7 +81,7 @@ func (l *gCurd) generateWebModelDictOptions(ctx context.Context, in *CurdPreview
}
}
dictTypeIds = convert.UniqueSliceInt64(dictTypeIds)
dictTypeIds = convert.UniqueSlice(dictTypeIds)
if len(dictTypeIds) == 0 {
options["has"] = false
return options, nil

View File

@@ -108,8 +108,7 @@ func GenJoinSelect(ctx context.Context, entity interface{}, masterDao interface{
continue
}
}
return gstr.Implode(",", convert.UniqueSliceString(tmpFields)), nil
return gstr.Implode(",", convert.UniqueSlice(tmpFields)), nil
}
// GenSelect 生成select
@@ -144,8 +143,7 @@ func GenSelect(ctx context.Context, entity interface{}, dao interface{}) (allFie
continue
}
}
return gstr.Implode(",", convert.UniqueSliceString(tmpFields)), nil
return gstr.Implode(",", convert.UniqueSlice(tmpFields)), nil
}
// GetPkField 获取dao实例中的主键名称
@@ -163,7 +161,6 @@ func GetPkField(ctx context.Context, dao daoInstance) (string, error) {
return field.Name, nil
}
}
return "", errors.New("no primary key")
}

View File

@@ -38,7 +38,6 @@ func FilterAuth(m *gdb.Model) *gdb.Model {
if !needAuth {
return m
}
return m.Handler(FilterAuthWithField(filterField))
}

View File

@@ -72,20 +72,19 @@ func WhoisLocation(ctx context.Context, ip string) (*IpLocationData, error) {
return nil, err
}
var whoisData *WhoisRegionData
if err = gconv.Struct([]byte(str), &whoisData); err != nil {
var who *WhoisRegionData
if err = gconv.Struct([]byte(str), &who); err != nil {
return nil, err
}
return &IpLocationData{
Ip: whoisData.Ip,
Region: whoisData.Addr,
Province: whoisData.Pro,
ProvinceCode: gconv.Int64(whoisData.ProCode),
City: whoisData.City,
CityCode: gconv.Int64(whoisData.CityCode),
Area: whoisData.Region,
AreaCode: gconv.Int64(whoisData.RegionCode),
Ip: who.Ip,
Region: who.Addr,
Province: who.Pro,
ProvinceCode: gconv.Int64(who.ProCode),
City: who.City,
CityCode: gconv.Int64(who.CityCode),
Area: who.Region,
AreaCode: gconv.Int64(who.RegionCode),
}, nil
}
@@ -159,7 +158,6 @@ func GetLocation(ctx context.Context, ip string) (data *IpLocationData, err erro
}
cacheMap.Set(ip, data)
}
return
}
@@ -234,6 +232,5 @@ func GetClientIp(r *ghttp.Request) string {
if gstr.Contains(ip, ", ") {
ip = gstr.StrTillEx(ip, ", ")
}
return ip
}

View File

@@ -52,7 +52,6 @@ func ParseSimpleRegion(ctx context.Context, id int64, spilt ...string) (string,
}
return ParseRegion(ctx, ids[0], ids[1], id, spilt...)
}
return "", gerror.New("currently, it is only supported to regional areas")
}
@@ -104,6 +103,5 @@ func ParseRegion(ctx context.Context, province int64, city int64, county int64,
if province > 0 && city > 0 {
return provinceName.String() + sp + cityName.String(), nil
}
return provinceName.String(), nil
}

View File

@@ -22,41 +22,42 @@ import (
// ClientConfig 客户端配置
type ClientConfig struct {
Addr string
Auth *AuthMeta
Timeout time.Duration
ConnectInterval time.Duration
MaxConnectCount uint
ConnectCount uint
AutoReconnect bool
LoginEvent CallbackEvent
CloseEvent CallbackEvent
Addr string // 连接地址
Auth *AuthMeta // 认证元数据
Timeout time.Duration // 连接超时时间
ConnectInterval time.Duration // 重连时间间隔
MaxConnectCount uint // 最大重连次数0不限次数
ConnectCount uint // 已重连次数
AutoReconnect bool // 是否开启自动重连
LoginEvent CallbackEvent // 登录成功事件
CloseEvent CallbackEvent // 连接关闭事件
}
// Client 客户端
type Client struct {
Ctx context.Context
Logger *glog.Logger
IsLogin bool // 是否已登录
addr string
auth *AuthMeta
rpc *Rpc
timeout time.Duration
connectInterval time.Duration
maxConnectCount uint
connectCount uint
autoReconnect bool
loginEvent CallbackEvent
closeEvent CallbackEvent
sync.Mutex
heartbeat int64
routers map[string]RouterHandler
conn *gtcp.Conn
wg sync.WaitGroup
closeFlag bool // 关闭标签,关闭以后可以重连
stopFlag bool // 停止标签,停止以后不能重连
Ctx context.Context // 上下文
Logger *glog.Logger // 日志处理器
IsLogin bool // 是否已登录
addr string // 连接地址
auth *AuthMeta // 认证元数据
rpc *Rpc // rpc协议支持
timeout time.Duration // 连接超时时间
connectInterval time.Duration // 重连时间间隔
maxConnectCount uint // 最大重连次数0不限次数
connectCount uint // 已重连次数
autoReconnect bool // 是否开启自动重连
loginEvent CallbackEvent // 登录成功事件
closeEvent CallbackEvent // 连接关闭事件
sync.Mutex // 状态锁
heartbeat int64 // 心跳
routers map[string]RouterHandler // 已注册的路由
conn *gtcp.Conn // 连接对象
wg sync.WaitGroup // 状态控制
closeFlag bool // 关闭标签,关闭以后可以重连
stopFlag bool // 停止标签,停止以后不能重连
}
// NewClient 初始化一个tcp客户端
func NewClient(config *ClientConfig) (client *Client, err error) {
client = new(Client)
@@ -110,7 +111,7 @@ func NewClient(config *ClientConfig) (client *Client, err error) {
return
}
// Start 启动
// Start 启动tcp连接
func (client *Client) Start() (err error) {
client.Lock()
defer client.Unlock()
@@ -133,7 +134,6 @@ func (client *Client) Start() (err error) {
simple.SafeGo(client.Ctx, func(ctx context.Context) {
client.connect()
})
return
}
@@ -165,6 +165,7 @@ func (client *Client) RegisterRouter(routers map[string]RouterHandler) (err erro
return
}
// dial
func (client *Client) dial() *gtcp.Conn {
for {
conn, err := gtcp.NewConn(client.addr, client.timeout)
@@ -218,6 +219,7 @@ reconnect:
client.startCron()
}
// read
func (client *Client) read() {
simple.SafeGo(client.Ctx, func(ctx context.Context) {
defer func() {
@@ -347,7 +349,6 @@ func (client *Client) Write(data interface{}) error {
return gerror.Newf("client json message pointer required: %+v", data)
}
msg := &Message{Router: msgType.Elem().Name(), Data: data}
return SendPkg(client.conn, msg)
}
@@ -379,7 +380,6 @@ func (client *Client) RpcRequest(ctx context.Context, data interface{}) (res int
err = gerror.New("traceID is required")
return
}
return client.rpc.Request(key, func() {
_ = client.Write(data)
})

View File

@@ -13,21 +13,24 @@ import (
"hotgo/internal/consts"
)
// getCronKey 生成客户端定时任务名称
func (client *Client) getCronKey(s string) string {
return fmt.Sprintf("tcp.client_%s_%s:%s", s, client.auth.Group, client.auth.Name)
}
// stopCron 停止定时任务
func (client *Client) stopCron() {
for _, v := range gcron.Entries() {
gcron.Remove(v.Name)
}
}
// startCron 启动定时任务
func (client *Client) startCron() {
// 心跳超时检查
if gcron.Search(client.getCronKey(consts.TCPCronHeartbeatVerify)) == nil {
_, _ = gcron.AddSingleton(client.Ctx, "@every 600s", func(ctx context.Context) {
if client.heartbeat < gtime.Timestamp()-600 {
if client.heartbeat < gtime.Timestamp()-consts.TCPHeartbeatTimeout {
client.Logger.Debugf(client.Ctx, "client heartbeat timeout, about to reconnect..")
client.Destroy()
}

View File

@@ -37,6 +37,7 @@ func (client *Client) serverLogin() {
}
}
// onResponseServerLogin 接收服务登陆响应结果
func (client *Client) onResponseServerLogin(ctx context.Context, args ...interface{}) {
var in *msgin.ResponseServerLogin
if err := gconv.Scan(args[0], &in); err != nil {
@@ -58,6 +59,7 @@ func (client *Client) onResponseServerLogin(ctx context.Context, args ...interfa
}
}
// onResponseServerHeartbeat 接收心跳响应结果
func (client *Client) onResponseServerHeartbeat(ctx context.Context, args ...interface{}) {
var in *msgin.ResponseServerHeartbeat
if err := gconv.Scan(args[0], &in); err != nil {

View File

@@ -19,6 +19,7 @@ type AuthMeta struct {
EndAt *gtime.Time `json:"-"`
}
// Context tcp上下文
type Context struct {
Conn *gtcp.Conn `json:"conn"`
Auth *AuthMeta `json:"auth"` // 认证元数据

View File

@@ -14,8 +14,10 @@ import (
"github.com/gogf/gf/v2/util/gconv"
)
var GoPool = grpool.New(100)
// GoPool 初始化一个协程池,用于处理消息处理
var GoPool = grpool.New(20)
// RouterHandler 路由消息处理器
type RouterHandler func(ctx context.Context, args ...interface{})
// Message 路由消息
@@ -24,6 +26,7 @@ type Message struct {
Data interface{} `json:"data"`
}
// SendPkg 打包发送的数据包
func SendPkg(conn *gtcp.Conn, message *Message) error {
b, err := json.Marshal(message)
if err != nil {
@@ -32,6 +35,7 @@ func SendPkg(conn *gtcp.Conn, message *Message) error {
return conn.SendPkg(b)
}
// RecvPkg 解包
func RecvPkg(conn *gtcp.Conn) (*Message, error) {
if data, err := conn.RecvPkg(); err != nil {
return nil, err
@@ -58,7 +62,6 @@ func MsgPkg(data interface{}, auth *AuthMeta, traceID string) string {
if msg == nil {
return ""
}
return msg.TraceID
}

View File

@@ -22,6 +22,7 @@ type Rpc struct {
callbacks map[string]RpcRespFunc
}
// RpcResp 响应结构
type RpcResp struct {
res interface{}
err error
@@ -29,6 +30,7 @@ type RpcResp struct {
type RpcRespFunc func(resp interface{}, err error)
// NewRpc 初始化一个rpc协议
func NewRpc(ctx context.Context) *Rpc {
return &Rpc{
ctx: ctx,
@@ -57,7 +59,6 @@ func (r *Rpc) HandleMsg(ctx context.Context, cancel context.CancelFunc, data int
})
return true
}
return false
}
@@ -99,7 +100,7 @@ func (r *Rpc) Request(callId string, send func()) (res interface{}, err error) {
<-waitCh
select {
case <-time.After(consts.TCPRpcTimeout):
case <-time.After(time.Second * consts.TCPRpcTimeout):
err = gerror.New("rpc response timeout")
return
case got := <-resCh:

View File

@@ -19,35 +19,38 @@ import (
"time"
)
// ClientConn 连接到tcp服务器的客户端对象
type ClientConn struct {
Conn *gtcp.Conn
Auth *AuthMeta
heartbeat int64
Conn *gtcp.Conn // 连接对象
Auth *AuthMeta // 认证元数据
heartbeat int64 // 心跳
}
// ServerConfig tcp服务器配置
type ServerConfig struct {
Name string // 服务名称
Addr string // 监听地址
}
// Server tcp服务器对象结构
type Server struct {
Ctx context.Context
Logger *glog.Logger
addr string
name string
rpc *Rpc
ln *gtcp.Server
wgLn sync.WaitGroup
mutex sync.Mutex
closeFlag bool
clients map[string]*ClientConn // 已登录的认证客户端
mutexConns sync.Mutex
wgConns sync.WaitGroup
cronRouters map[string]RouterHandler // 路由
queueRouters map[string]RouterHandler
authRouters map[string]RouterHandler
Ctx context.Context // 上下文
Logger *glog.Logger // 日志处理器
addr string // 连接地址
name string // 服务器名称
rpc *Rpc // rpc协议
ln *gtcp.Server // tcp服务器
wgLn sync.WaitGroup // 状态控制主要用于tcp服务器能够按流程启动退出
mutex sync.Mutex // 服务器状态锁
closeFlag bool // 服务关闭标签
clients map[string]*ClientConn // 已登录的认证客户端
mutexConns sync.Mutex // 连接锁,主要用于客户端上下线
cronRouters map[string]RouterHandler // 定时任务路由
queueRouters map[string]RouterHandler // 队列路由
authRouters map[string]RouterHandler // 任务路由
}
// NewServer 初始一个tcp服务器对象
func NewServer(config *ServerConfig) (server *Server, err error) {
if config == nil {
err = gerror.New("config is nil")
@@ -84,6 +87,7 @@ func NewServer(config *ServerConfig) (server *Server, err error) {
return
}
// accept
func (server *Server) accept(conn *gtcp.Conn) {
defer func() {
server.mutexConns.Lock()
@@ -262,6 +266,7 @@ func (server *Server) RegisterQueueRouter(routers map[string]RouterHandler) {
}
}
// Listen 监听服务
func (server *Server) Listen() (err error) {
server.wgLn.Add(1)
defer server.wgLn.Done()
@@ -283,7 +288,6 @@ func (server *Server) Close() {
}
server.clients = nil
server.mutexConns.Unlock()
server.wgConns.Wait()
if server.ln != nil {
_ = server.ln.Close()

View File

@@ -13,16 +13,19 @@ import (
"hotgo/internal/consts"
)
// getCronKey 生成服务端定时任务名称
func (server *Server) getCronKey(s string) string {
return fmt.Sprintf("tcp.server_%s_%s", s, server.name)
}
// stopCron 停止定时任务
func (server *Server) stopCron() {
for _, v := range gcron.Entries() {
gcron.Remove(v.Name)
}
}
// startCron 启动定时任务
func (server *Server) startCron() {
// 心跳超时检查
if gcron.Search(server.getCronKey(consts.TCPCronHeartbeatVerify)) == nil {
@@ -31,7 +34,7 @@ func (server *Server) startCron() {
return
}
for _, client := range server.clients {
if client.heartbeat < gtime.Timestamp()-300 {
if client.heartbeat < gtime.Timestamp()-consts.TCPHeartbeatTimeout {
_ = client.Conn.Close()
server.Logger.Debugf(server.Ctx, "client heartbeat timeout, close conn. auth:%+v", client.Auth)
}

View File

@@ -17,6 +17,7 @@ import (
"hotgo/utility/convert"
)
// onServerLogin 处理客户端登录
func (server *Server) onServerLogin(ctx context.Context, args ...interface{}) {
var (
in = new(msgin.ServerLogin)
@@ -137,6 +138,7 @@ func (server *Server) onServerLogin(ctx context.Context, args ...interface{}) {
_ = server.Write(user.Conn, res)
}
// onServerHeartbeat 处理客户端心跳
func (server *Server) onServerHeartbeat(ctx context.Context, args ...interface{}) {
var (
in *msgin.ServerHeartbeat

View File

@@ -93,7 +93,6 @@ func (h *aliPay) Notify(ctx context.Context, in payin.NotifyInp) (res *payin.Not
res.OutTradeNo = notify.OutTradeNo
res.PayAt = notify.GmtPayment
res.ActualAmount = gconv.Float64(notify.ReceiptAmount)
return
}
@@ -115,7 +114,6 @@ func (h *aliPay) CreateOrder(ctx context.Context, in payin.CreateOrderInp) (res
default:
err = gerror.Newf("暂未支持的交易方式:%v", in.Pay.TradeType)
}
return
}

View File

@@ -44,7 +44,6 @@ func New(name ...string) PayClient {
default:
panic(fmt.Sprintf("暂不支持的支付方式:%v", payType))
}
return client
}

View File

@@ -75,7 +75,6 @@ func (h *qqPay) Notify(ctx context.Context, in payin.NotifyInp) (res *payin.Noti
res.OutTradeNo = notify.OutTradeNo
res.PayAt = gtime.New(notify.TimeEnd)
res.ActualAmount = gconv.Float64(notify.CouponFee) / 100 // 用户本次交易中,实际支付的金额 转为元,和系统内保持一至
return
}
@@ -119,7 +118,6 @@ func (h *qqPay) CreateOrder(ctx context.Context, in payin.CreateOrderInp) (res *
default:
err = gerror.Newf("暂未支持的交易方式:%v", in.Pay.TradeType)
}
return
}

View File

@@ -105,7 +105,6 @@ func (h *wxPay) Notify(ctx context.Context, in payin.NotifyInp) (res *payin.Noti
res.OutTradeNo = notify.OutTradeNo
res.PayAt = gtime.New(notify.SuccessTime)
res.ActualAmount = float64(notify.Amount.PayerTotal / 100) // 转为元,和系统内保持一至
return
}
@@ -121,7 +120,6 @@ func (h *wxPay) CreateOrder(ctx context.Context, in payin.CreateOrderInp) (res *
default:
err = gerror.Newf("暂未支持的交易方式:%v", in.Pay.TradeType)
}
return
}

View File

@@ -69,5 +69,4 @@ func consumerListen(ctx context.Context, job consumerStrategy) {
}); listenErr != nil {
g.Log().Fatalf(ctx, "消费队列:%s 监听失败, err:%+v", topic, listenErr)
}
}

View File

@@ -11,7 +11,8 @@ import (
"github.com/Shopify/sarama"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"hotgo/utility/signal"
"github.com/gogf/gf/v2/os/gproc"
"os"
"time"
)
@@ -109,14 +110,13 @@ func (r *KafkaMq) ListenReceiveMsgDo(topic string, receiveDo func(mqMsg MqMsg))
<-consumer.ready
g.Log().Debug(ctx, "kafka consumer up and running!...")
signal.AppDefer(func() {
gproc.AddSigHandlerShutdown(func(sig os.Signal) {
g.Log().Debug(ctx, "kafka consumer close...")
cancel()
if err = r.consumerIns.Close(); err != nil {
g.Log().Fatalf(ctx, "kafka Error closing client, err:%+v", err)
}
})
return
}
@@ -203,7 +203,7 @@ func doRegisterKafkaProducer(connOpt KafkaConfig, mqIns *KafkaMq) (err error) {
return
}
signal.AppDefer(func() {
gproc.AddSigHandlerShutdown(func(sig os.Signal) {
g.Log().Debug(ctx, "kafka producer AsyncClose...")
mqIns.producerIns.AsyncClose()
})

View File

@@ -0,0 +1,22 @@
package storager
import (
"context"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"hotgo/internal/model"
)
var config *model.UploadConfig
func SetConfig(c *model.UploadConfig) {
config = c
}
func GetConfig() *model.UploadConfig {
return config
}
func GetModel(ctx context.Context) *gdb.Model {
return g.Model("sys_attachment").Ctx(ctx)
}

View File

@@ -0,0 +1,160 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/text/gstr"
"io"
"path"
)
// 文件归属分类
const (
KindImg = "images" // 图片
KindDoc = "document" // 文档
KindAudio = "audio" // 音频
KindVideo = "video" // 视频
KindOther = "other" // 其他
)
var (
// 图片类型
imgType = g.MapStrStr{
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
"cr2": "image/x-canon-cr2",
"tif": "image/tiff",
"bmp": "image/bmp",
"heif": "image/heif",
"jxr": "image/vnd.ms-photo",
"psd": "image/vnd.adobe.photoshop",
"ico": "image/vnd.microsoft.icon",
"dwg": "image/vnd.dwg",
}
// 文档类型
docType = g.MapStrStr{
"doc": "application/msword",
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"xls": "application/vnd.ms-excel",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"ppt": "application/vnd.ms-powerpoint",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
}
// 音频类型
audioType = g.MapStrStr{
"mid": "audio/midi",
"mp3": "audio/mpeg",
"m4a": "audio/mp4",
"ogg": "audio/ogg",
"flac": "audio/x-flac",
"wav": "audio/x-wav",
"amr": "audio/amr",
"aac": "audio/aac",
"aiff": "audio/x-aiff",
}
// 视频类型
videoType = g.MapStrStr{
"mp4": "video/mp4",
"m4v": "video/x-m4v",
"mkv": "video/x-matroska",
"webm": "video/webm",
"mov": "video/quicktime",
"avi": "video/x-msvideo",
"wmv": "video/x-ms-wmv",
"mpg": "video/mpeg",
"flv": "video/x-flv",
"3gp": "video/3gpp",
}
)
// IsImgType 判断是否为图片
func IsImgType(ext string) bool {
_, ok := imgType[ext]
return ok
}
// IsDocType 判断是否为文档
func IsDocType(ext string) bool {
_, ok := docType[ext]
return ok
}
// IsAudioType 判断是否为音频
func IsAudioType(ext string) bool {
_, ok := audioType[ext]
return ok
}
// IsVideoType 判断是否为视频
func IsVideoType(ext string) bool {
_, ok := videoType[ext]
return ok
}
// GetImgType 获取图片类型
func GetImgType(ext string) (string, error) {
if mime, ok := imgType[ext]; ok {
return mime, nil
}
return "", gerror.New("Invalid image type")
}
// GetFileType 获取文件类型
func GetFileType(ext string) (string, error) {
if mime, ok := imgType[ext]; ok {
return mime, nil
}
if mime, ok := docType[ext]; ok {
return mime, nil
}
if mime, ok := audioType[ext]; ok {
return mime, nil
}
if mime, ok := videoType[ext]; ok {
return mime, nil
}
return "", gerror.Newf("Invalid file type:%v", ext)
}
// GetFileKind 获取文件所属分类
func GetFileKind(ext string) string {
if _, ok := imgType[ext]; ok {
return KindImg
}
if _, ok := docType[ext]; ok {
return KindDoc
}
if _, ok := audioType[ext]; ok {
return KindAudio
}
if _, ok := videoType[ext]; ok {
return KindVideo
}
return KindOther
}
// Ext 获取文件后缀
func Ext(baseName string) string {
return gstr.StrEx(path.Ext(baseName), ".")
}
// UploadFileByte 获取上传文件的byte
func UploadFileByte(file *ghttp.UploadFile) ([]byte, error) {
open, err := file.Open()
if err != nil {
return nil, err
}
return io.ReadAll(open)
}

View File

@@ -0,0 +1,12 @@
package storager
// FileMeta 文件元数据
type FileMeta struct {
Filename string // 文件名称
Size int64 // 文件大小
Kind string // 文件所属分类
MetaType string // 文件类型
NaiveType string // NaiveUI类型
Ext string // 文件后缀名
Md5 string // 文件hash
}

View File

@@ -0,0 +1,228 @@
package storager
import (
"context"
"fmt"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"github.com/gogf/gf/v2/util/grand"
"hotgo/internal/consts"
"hotgo/internal/library/contexts"
"hotgo/internal/model/entity"
"hotgo/utility/encrypt"
"hotgo/utility/url"
"hotgo/utility/validate"
"strconv"
"strings"
)
// UploadDrive 存储驱动
type UploadDrive interface {
// Upload 上传
Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error)
}
// New 初始化存储驱动
func New(name ...string) UploadDrive {
var (
driveType = consts.UploadDriveLocal
drive UploadDrive
)
if len(name) > 0 && name[0] != "" {
driveType = name[0]
}
switch driveType {
case consts.UploadDriveLocal:
drive = &LocalDrive{}
case consts.UploadDriveUCloud:
drive = &UCloudDrive{}
case consts.UploadDriveCos:
drive = &CosDrive{}
case consts.UploadDriveOss:
drive = &OssDrive{}
case consts.UploadDriveQiNiu:
drive = &QiNiuDrive{}
default:
panic(fmt.Sprintf("暂不支持的存储驱动:%v", driveType))
}
return drive
}
// DoUpload 上传入口
func DoUpload(ctx context.Context, file *ghttp.UploadFile, typ int) (result *entity.SysAttachment, err error) {
if file == nil {
err = gerror.New("文件必须!")
return
}
meta, err := GetFileMeta(file)
if err != nil {
return
}
if _, err = GetFileType(meta.Ext); err != nil {
return
}
switch typ {
case consts.UploadTypeFile:
if config.FileSize > 0 && meta.Size > config.FileSize*1024*1024 {
err = gerror.Newf("文件大小不能超过%vMB", config.FileSize)
return
}
case consts.UploadTypeImage:
if !IsImgType(meta.Ext) {
err = gerror.New("上传的文件不是图片")
return
}
if config.ImageSize > 0 && meta.Size > config.ImageSize*1024*1024 {
err = gerror.Newf("图片大小不能超过%vMB", config.ImageSize)
return
}
case consts.UploadTypeDoc:
if !IsDocType(meta.Ext) {
err = gerror.New("上传的文件不是文档")
return
}
case consts.UploadTypeAudio:
if !IsAudioType(meta.Ext) {
err = gerror.New("上传的文件不是音频")
return
}
case consts.UploadTypeVideo:
if !IsVideoType(meta.Ext) {
err = gerror.New("上传的文件不是视频")
return
}
default:
err = gerror.Newf("无效的上传类型:%v", typ)
return
}
result, err = hasFile(ctx, meta.Md5)
if err != nil {
return
}
if result != nil {
return
}
// 上传到驱动
fullPath, err := New(config.Drive).Upload(ctx, file)
if err != nil {
return
}
// 写入附件记录
return write(ctx, meta, fullPath)
}
// LastUrl 根据驱动获取最终文件访问地址
func LastUrl(ctx context.Context, fullPath, drive string) string {
if validate.IsURL(fullPath) {
return fullPath
}
switch drive {
case consts.UploadDriveLocal:
return url.GetAddr(ctx) + "/" + fullPath
case consts.UploadDriveUCloud:
return config.UCloudEndpoint + "/" + fullPath
case consts.UploadDriveCos:
return config.CosBucketURL + "/" + fullPath
case consts.UploadDriveOss:
return config.OssBucketURL + "/" + fullPath
case consts.UploadDriveQiNiu:
return config.QiNiuDomain + "/" + fullPath
default:
return fullPath
}
}
// GetFileMeta 获取上传文件元数据
func GetFileMeta(file *ghttp.UploadFile) (meta *FileMeta, err error) {
meta = new(FileMeta)
meta.Filename = file.Filename
meta.Size = file.Size
meta.Ext = Ext(file.Filename)
meta.Kind = GetFileKind(meta.Ext)
meta.MetaType, err = GetFileType(meta.Ext)
if err != nil {
return
}
// 兼容naiveUI
naiveType := "text/plain"
if IsImgType(Ext(file.Filename)) {
naiveType = ""
}
meta.NaiveType = naiveType
// 文件hash
b, err := UploadFileByte(file)
if err != nil {
return
}
meta.Md5 = encrypt.Md5ToString(gconv.String(encrypt.Hash32(b)))
return
}
// GenFullPath 根据目录和文件类型生成一个绝对地址
func GenFullPath(basePath, ext string) string {
fileName := strconv.FormatInt(gtime.TimestampNano(), 36) + grand.S(6)
fileName = fileName + ext
return basePath + gtime.Date() + "/" + strings.ToLower(fileName)
}
// write 写入附件记录
func write(ctx context.Context, meta *FileMeta, fullPath string) (models *entity.SysAttachment, err error) {
models = &entity.SysAttachment{
Id: 0,
AppId: contexts.GetModule(ctx),
MemberId: contexts.GetUserId(ctx),
Drive: config.Drive,
Size: meta.Size,
Path: fullPath,
FileUrl: fullPath,
Name: meta.Filename,
Kind: meta.Kind,
MetaType: meta.MetaType,
NaiveType: meta.NaiveType,
Ext: meta.Ext,
Md5: meta.Md5,
Status: consts.StatusEnabled,
}
id, err := GetModel(ctx).Data(models).InsertAndGetId()
if err != nil {
return
}
models.Id = id
return
}
// hasFile 检查附件是否存在
func hasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
if err = GetModel(ctx).Where("md5", md5).Scan(&res); err != nil {
err = gerror.Wrap(err, "检查文件hash时出现错误")
return
}
if res == nil {
return
}
// 只有在上传时才会检查md5值如果附件存在则更新最后上传时间保证上传列表更新显示在最前面
if res.Id > 0 {
_, _ = GetModel(ctx).WherePri(res.Id).Data(g.Map{
"status": consts.StatusEnabled,
"updated_at": gtime.Now(),
}).Update()
}
return
}

View File

@@ -0,0 +1,42 @@
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
)
// CosDrive 腾讯云cos驱动
type CosDrive struct {
}
// Upload 上传到腾讯云cos对象存储
func (d *CosDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.CosPath == "" {
err = gerror.New("COS存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
URL, _ := url.Parse(config.CosBucketURL)
client := cos.NewClient(&cos.BaseURL{BucketURL: URL}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: config.CosSecretId,
SecretKey: config.CosSecretKey,
},
})
fullPath = GenFullPath(config.UCloudPath, gfile.Ext(file.Filename))
_, err = client.Object.Put(ctx, fullPath, f2, nil)
return
}

View File

@@ -0,0 +1,42 @@
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"strings"
)
// LocalDrive 本地驱动
type LocalDrive struct {
}
// Upload 上传到本地
func (d *LocalDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
var (
sp = g.Cfg().MustGet(ctx, "server.serverRoot")
nowDate = gtime.Date()
)
if sp.IsEmpty() {
err = gerror.New("本地上传驱动必须配置静态路径!")
return
}
if config.LocalPath == "" {
err = gerror.New("本地上传驱动必须配置本地存储路径!")
return
}
// 包含静态文件夹的路径
fullDirPath := strings.Trim(sp.String(), "/") + "/" + config.LocalPath + nowDate
fileName, err := file.Save(fullDirPath, true)
if err != nil {
return
}
// 不含静态文件夹的路径
fullPath = config.LocalPath + nowDate + "/" + fileName
return
}

View File

@@ -0,0 +1,42 @@
package storager
import (
"context"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
)
// OssDrive 阿里云oss驱动
type OssDrive struct {
}
// Upload 上传到阿里云oss
func (d *OssDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.OssPath == "" {
err = gerror.New("OSS存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
client, err := oss.New(config.OssEndpoint, config.OssSecretId, config.OssSecretKey)
if err != nil {
return
}
bucket, err := client.Bucket(config.OssBucket)
if err != nil {
return
}
fullPath = GenFullPath(config.UCloudPath, gfile.Ext(file.Filename))
err = bucket.PutObject(fullPath, f2)
return
}

View File

@@ -0,0 +1,52 @@
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// QiNiuDrive 七牛云对象存储驱动
type QiNiuDrive struct {
}
// Upload 上传到七牛云对象存储
func (d *QiNiuDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.QiNiuPath == "" {
err = gerror.New("七牛云存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
putPolicy := storage.PutPolicy{
Scope: config.QiNiuBucket,
}
token := putPolicy.UploadToken(qbox.NewMac(config.QiNiuAccessKey, config.QiNiuSecretKey))
cfg := storage.Config{}
// 是否使用https域名
cfg.UseHTTPS = true
// 上传是否使用CDN上传加速
cfg.UseCdnDomains = false
// 空间对应的机房
cfg.Region, err = storage.GetRegion(config.QiNiuAccessKey, config.QiNiuBucket)
if err != nil {
return
}
fullPath = GenFullPath(config.UCloudPath, gfile.Ext(file.Filename))
err = storage.NewFormUploader(&cfg).Put(ctx, &storage.PutRet{}, token, fullPath, f2, file.Size, &storage.PutExtra{})
return
}

View File

@@ -0,0 +1,45 @@
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
upload "github.com/ufilesdk-dev/ufile-gosdk"
)
// UCloudDrive UCloud对象存储驱动
type UCloudDrive struct {
}
// Upload 上传到UCloud对象存储
func (d *UCloudDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.UCloudPath == "" {
err = gerror.New("UCloud存储驱动必须配置存储路径!")
return
}
client, err := upload.NewFileRequest(&upload.Config{
PublicKey: config.UCloudPublicKey,
PrivateKey: config.UCloudPrivateKey,
BucketHost: config.UCloudBucketHost,
BucketName: config.UCloudBucketName,
FileHost: config.UCloudFileHost,
Endpoint: config.UCloudEndpoint,
VerifyUploadMD5: false,
}, nil)
if err != nil {
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
fullPath = GenFullPath(config.UCloudPath, gfile.Ext(file.Filename))
err = client.IOPut(f2, fullPath, "")
return
}