add golang golint, workerflow

This commit is contained in:
aceld 2024-04-15 17:50:02 +08:00
parent 142622dfc5
commit 955bed4804
70 changed files with 806 additions and 757 deletions

20
.github/workflows/reviewdog.yml vendored Normal file
View File

@ -0,0 +1,20 @@
name: reviewdog
on: [pull_request]
jobs:
golangci-lint:
runs-on: ubuntu-latest
name: runner / golangci-lint
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: golangci-lint
uses: reviewdog/action-golangci-lint@v2
with:
github_token: ${{ secrets.github_token }}
# Change reviewdog reporter if you need [github-pr-check,github-check,github-pr-review].
reporter: github-pr-review
# Report all results.
filter_mode: nofilter
# Exit with 1 when it finds at least one finding.
fail_on_error: true
#golangci_lint_flags

28
.golangci.yaml Normal file
View File

@ -0,0 +1,28 @@
run:
timeout: 30m
skip-dirs:
- test
linters:
disable-all: true
enable:
#- unused
- ineffassign
- goimports
- gofmt
- misspell
- unparam
- unconvert
- govet
# - errcheck
- staticcheck
linters-settings:
staticcheck:
go: "1.17"
checks:
- "all"
- "-SA1019"
unused:
go: "1.17"

View File

@ -2,93 +2,100 @@ package common
import "time" import "time"
// 用户生成KisId的字符串前缀 // Prefix string for generating KisId by users
const ( const (
KisIdTypeFlow = "flow" KisIDTypeFlow = "flow" // KisId type for Flow
KisIdTypeConnector = "conn" KisIDTypeConnector = "conn" // KisId type for Connector
KisIdTypeFunction = "func" KisIDTypeFunction = "func" // KisId type for Function
KisIdTypeGlobal = "global" KisIDTypeGlobal = "global" // KisId type for Global
KisIdJoinChar = "-" KisIDJoinChar = "-" // Joining character for KisId
) )
const ( const (
// FunctionIdFirstVirtual 为首结点Function上一层虚拟的Function ID // FunctionIDFirstVirtual is the virtual Function ID for the first node Function
FunctionIdFirstVirtual = "FunctionIdFirstVirtual" FunctionIDFirstVirtual = "FunctionIDFirstVirtual"
// FunctionIdLastVirtual 为尾结点Function下一层虚拟的Function ID // FunctionIDLastVirtual is the virtual Function ID for the last node Function
FunctionIdLastVirtual = "FunctionIdLastVirtual" FunctionIDLastVirtual = "FunctionIDLastVirtual"
) )
// KisMode represents the mode of KisFunction
type KisMode string type KisMode string
const ( const (
// V 为校验特征的KisFunction, 主要进行数据的过滤,验证,字段梳理,幂等等前置数据处理 // V is for Verify, which mainly performs data filtering, validation, field sorting, idempotence, etc.
V KisMode = "Verify" V KisMode = "Verify"
// S 为存储特征的KisFunction, S会通过KisConnector进行将数据进行存储. S Function 会通过KisConnector进行数据存储,具备相同Connector的Function在逻辑上可以进行并流 // S is for Save, S Function will store data through KisConnector. Functions with the same Connector can logically merge.
S KisMode = "Save" S KisMode = "Save"
// L 为加载特征的KisFunctionL会通过KisConnector进行数据加载L Function 会通过KisConnector进行数据读取具备相同Connector的Function可以从逻辑上与对应的S Function进行并流 // L is for Load, L Function will load data through KisConnector. Functions with the same Connector can logically merge with corresponding S Function.
L KisMode = "Load" L KisMode = "Load"
// C 为计算特征的KisFunction, 可以生成新的字段,计算新的值,进行数据的聚合,分析等 // C is for Calculate, which can generate new fields, calculate new values, and perform data aggregation, analysis, etc.
C KisMode = "Calculate" C KisMode = "Calculate"
// E 为扩展特征的KisFunction作为流式计算的自定义特征Function也同时是KisFlow当前流中的最后一个Function概念类似Sink。 // E is for Expand, which serves as a custom feature Function for stream computing and is also the last Function in the current KisFlow, similar to Sink.
E KisMode = "Expand" E KisMode = "Expand"
) )
/* // KisOnOff Whether to enable the Flow
是否启动Flow
*/
type KisOnOff int type KisOnOff int
const ( const (
FlowEnable KisOnOff = 1 // 启动 // FlowEnable Enabled
FlowDisable KisOnOff = 0 // 不启动 FlowEnable KisOnOff = 1
// FlowDisable Disabled
FlowDisable KisOnOff = 0
) )
// KisConnType represents the type of KisConnector
type KisConnType string type KisConnType string
const ( const (
// REDIS is the type of Redis
REDIS KisConnType = "redis" REDIS KisConnType = "redis"
// MYSQL is the type of MySQL
MYSQL KisConnType = "mysql" MYSQL KisConnType = "mysql"
// KAFKA is the type of Kafka
KAFKA KisConnType = "kafka" KAFKA KisConnType = "kafka"
TIDB KisConnType = "tidb" // TIDB is the type of TiDB
ES KisConnType = "es" TIDB KisConnType = "tidb"
// ES is the type of Elasticsearch
ES KisConnType = "es"
) )
// cache // cache
const ( const (
// DeFaultFlowCacheCleanUp KisFlow中Flow对象Cache缓存默认的清理内存时间 // DeFaultFlowCacheCleanUp is the default cleanup time for Cache in KisFlow's Flow object Cache
DeFaultFlowCacheCleanUp = 5 // 单位 min DeFaultFlowCacheCleanUp = 5 // unit: min
// DefaultExpiration 默认GoCahce时间 ,永久保存 // DefaultExpiration is the default time for GoCahce, permanent storage
DefaultExpiration time.Duration = 0 DefaultExpiration time.Duration = 0
) )
// metrics // metrics
const ( const (
METRICS_ROUTE string = "/metrics" MetricsRoute string = "/metrics"
LABEL_FLOW_NAME string = "flow_name" LabelFlowName string = "flow_name"
LABEL_FLOW_ID string = "flow_id" LabelFlowID string = "flow_id"
LABEL_FUNCTION_NAME string = "func_name" LabelFunctionName string = "func_name"
LABEL_FUNCTION_MODE string = "func_mode" LabelFunctionMode string = "func_mode"
COUNTER_KISFLOW_DATA_TOTAL_NAME string = "kisflow_data_total" CounterKisflowDataTotalName string = "kisflow_data_total"
COUNTER_KISFLOW_DATA_TOTAL_HELP string = "KisFlow全部Flow的数据总量" CounterKisflowDataTotalHelp string = "Total data volume of all KisFlow Flows"
GANGE_FLOW_DATA_TOTAL_NAME string = "flow_data_total" GamgeFlowDataTotalName string = "flow_data_total"
GANGE_FLOW_DATA_TOTAL_HELP string = "KisFlow各个FlowID数据流的数据数量总量" GamgeFlowDataTotalHelp string = "Total data volume of each KisFlow FlowID data stream"
GANGE_FLOW_SCHE_CNTS_NAME string = "flow_schedule_cnts" GangeFlowScheCntsName string = "flow_schedule_cnts"
GANGE_FLOW_SCHE_CNTS_HELP string = "KisFlow各个FlowID被调度的次数" GangeFlowScheCntsHelp string = "Number of times each KisFlow FlowID is scheduled"
GANGE_FUNC_SCHE_CNTS_NAME string = "func_schedule_cnts" GangeFuncScheCntsName string = "func_schedule_cnts"
GANGE_FUNC_SCHE_CNTS_HELP string = "KisFlow各个Function被调度的次数" GangeFuncScheCntsHelp string = "Number of times each KisFlow Function is scheduled"
HISTOGRAM_FUNCTION_DURATION_NAME string = "func_run_duration" HistogramFunctionDurationName string = "func_run_duration"
HISTOGRAM_FUNCTION_DURATION_HELP string = "Function执行耗时" HistogramFunctionDurationHelp string = "Function execution time"
HISTOGRAM_FLOW_DURATION_NAME string = "flow_run_duration" HistogramFlowDurationName string = "flow_run_duration"
HISTOGRAM_FLOW_DURATION_HELP string = "Flow执行耗时" HistogramFlowDurationHelp string = "Flow execution time"
) )

View File

@ -1,14 +1,12 @@
package common package common
// KisRow 一行数据 // KisRow represents a single row of data
type KisRow interface{} type KisRow interface{}
// KisRowArr 一次业务的批量数据 // KisRowArr represents a batch of data for a single business operation
type KisRowArr []KisRow type KisRowArr []KisRow
/* // KisDataMap contains all the data carried by the current Flow
KisDataMap 当前Flow承载的全部数据 // key : Function ID where the data resides
key : 数据所在的Function ID // value: Corresponding KisRow
value: 对应的KisRow
*/
type KisDataMap map[string]KisRowArr type KisDataMap map[string]KisRowArr

View File

@ -1,36 +1,29 @@
package config package config
import ( import (
"errors"
"fmt" "fmt"
"github.com/aceld/kis-flow/common" "github.com/aceld/kis-flow/common"
) )
// KisConnConfig KisConnector 策略配置 // KisConnConfig describes the KisConnector strategy configuration
type KisConnConfig struct { type KisConnConfig struct {
//配置类型 KisType string `yaml:"kistype"` // Configuration type
KisType string `yaml:"kistype"` CName string `yaml:"cname"` // Unique descriptive identifier
//唯一描述标识 AddrString string `yaml:"addrs"` // Base storage medium address
CName string `yaml:"cname"` Type common.KisConnType `yaml:"type"` // Storage medium engine type: "Mysql", "Redis", "Kafka", etc.
//基础存储媒介地址 Key string `yaml:"key"` // Identifier for a single storage: Key name for Redis, Table name for Mysql, Topic name for Kafka, etc.
AddrString string `yaml:"addrs"` Params map[string]string `yaml:"params"` // Custom parameters in the configuration information
//存储媒介引擎类型"Mysql" "Redis" "Kafka"等
Type common.KisConnType `yaml:"type"` // NsFuncionID bound to storage reading
//一次存储的标识如Redis为Key名称、Mysql为Table名称,Kafka为Topic名称等
Key string `yaml:"key"`
//配置信息中的自定义参数
Params map[string]string `yaml:"params"`
//存储读取所绑定的NsFuncionID
Load []string `yaml:"load"` Load []string `yaml:"load"`
Save []string `yaml:"save"` Save []string `yaml:"save"`
} }
// NewConnConfig 创建一个KisConnector策略配置对象, 用于描述一个KisConnector信息 // NewConnConfig creates a KisConnector strategy configuration object, used to describe a KisConnector information
func NewConnConfig(cName string, addr string, t common.KisConnType, key string, param FParam) *KisConnConfig { func NewConnConfig(cName string, addr string, t common.KisConnType, key string, param map[string]string) *KisConnConfig {
strategy := new(KisConnConfig) strategy := new(KisConnConfig)
strategy.CName = cName strategy.CName = cName
strategy.AddrString = addr strategy.AddrString = addr
strategy.Type = t strategy.Type = t
strategy.Key = key strategy.Key = key
strategy.Params = param strategy.Params = param
@ -38,7 +31,7 @@ func NewConnConfig(cName string, addr string, t common.KisConnType, key string,
return strategy return strategy
} }
// WithFunc Connector与Function进行关系绑定 // WithFunc binds Connector to Function
func (cConfig *KisConnConfig) WithFunc(fConfig *KisFuncConfig) error { func (cConfig *KisConnConfig) WithFunc(fConfig *KisFuncConfig) error {
switch common.KisMode(fConfig.FMode) { switch common.KisMode(fConfig.FMode) {
@ -47,7 +40,7 @@ func (cConfig *KisConnConfig) WithFunc(fConfig *KisFuncConfig) error {
case common.L: case common.L:
cConfig.Load = append(cConfig.Load, fConfig.FName) cConfig.Load = append(cConfig.Load, fConfig.FName)
default: default:
return errors.New(fmt.Sprintf("Wrong KisMode %s", fConfig.FMode)) return fmt.Errorf("Wrong KisMode %s", fConfig.FMode)
} }
return nil return nil

View File

@ -2,13 +2,13 @@ package config
import "github.com/aceld/kis-flow/common" import "github.com/aceld/kis-flow/common"
// KisFlowFunctionParam 一个Flow配置中Function的Id及携带固定配置参数 // KisFlowFunctionParam represents the Id of a Function and carries fixed configuration parameters in a Flow configuration
type KisFlowFunctionParam struct { type KisFlowFunctionParam struct {
FuncName string `yaml:"fname"` //必须 FuncName string `yaml:"fname"` // Required
Params FParam `yaml:"params"` //选填,在当前Flow中Function定制固定配置参数 Params FParam `yaml:"params"` // Optional, custom fixed configuration parameters for the Function in the current Flow
} }
// KisFlowConfig 用户贯穿整条流式计算上下文环境的对象 // KisFlowConfig represents the object that spans the entire stream computing context environment
type KisFlowConfig struct { type KisFlowConfig struct {
KisType string `yaml:"kistype"` KisType string `yaml:"kistype"`
Status int `yaml:"status"` Status int `yaml:"status"`
@ -16,7 +16,7 @@ type KisFlowConfig struct {
Flows []KisFlowFunctionParam `yaml:"flows"` Flows []KisFlowFunctionParam `yaml:"flows"`
} }
// NewFlowConfig 创建一个Flow策略配置对象, 用于描述一个KisFlow信息 // NewFlowConfig creates a Flow strategy configuration object, used to describe a KisFlow information
func NewFlowConfig(flowName string, enable common.KisOnOff) *KisFlowConfig { func NewFlowConfig(flowName string, enable common.KisOnOff) *KisFlowConfig {
config := new(KisFlowConfig) config := new(KisFlowConfig)
config.FlowName = flowName config.FlowName = flowName
@ -27,7 +27,7 @@ func NewFlowConfig(flowName string, enable common.KisOnOff) *KisFlowConfig {
return config return config
} }
// AppendFunctionConfig 添加一个Function Config 到当前Flow中 // AppendFunctionConfig adds a Function Config to the current Flow
func (fConfig *KisFlowConfig) AppendFunctionConfig(params KisFlowFunctionParam) { func (fConfig *KisFlowConfig) AppendFunctionConfig(params KisFlowFunctionParam) {
fConfig.Flows = append(fConfig.Flows, params) fConfig.Flows = append(fConfig.Flows, params)
} }

View File

@ -6,24 +6,24 @@ import (
"github.com/aceld/kis-flow/log" "github.com/aceld/kis-flow/log"
) )
// FParam 在当前Flow中Function定制固定配置参数类型 // FParam represents the type for custom fixed configuration parameters for the Function in the current Flow
type FParam map[string]string type FParam map[string]string
// KisSource 表示当前Function的业务源 // KisSource represents the business source of the current Function
type KisSource struct { type KisSource struct {
Name string `yaml:"name"` // 本层Function的数据源描述 Name string `yaml:"name"` // Description of the data source for this layer Function
Must []string `yaml:"must"` // source必传字段 Must []string `yaml:"must"` // Required fields for the source
} }
// KisFuncOption 可选配置 // KisFuncOption represents optional configurations
type KisFuncOption struct { type KisFuncOption struct {
CName string `yaml:"cname"` // 连接器Connector名称 CName string `yaml:"cname"` // Connector name
RetryTimes int `yaml:"retry_times"` // 选填,Function调度重试(不包括正常调度)最大次数 RetryTimes int `yaml:"retry_times"` // Optional, maximum retry times for Function scheduling (excluding normal scheduling)
RetryDuration int `yaml:"return_duration"` // 选填,Function调度每次重试最大时间间隔(单位:ms) RetryDuration int `yaml:"return_duration"` // Optional, maximum time interval for each retry in Function scheduling (unit: ms)
Params FParam `yaml:"default_params"` // 选填,在当前Flow中Function定制固定配置参数 Params FParam `yaml:"default_params"` // Optional, custom fixed configuration parameters for the Function in the current Flow
} }
// KisFuncConfig 一个KisFunction策略配置 // KisFuncConfig represents a KisFunction strategy configuration
type KisFuncConfig struct { type KisFuncConfig struct {
KisType string `yaml:"kistype"` KisType string `yaml:"kistype"`
FName string `yaml:"fname"` FName string `yaml:"fname"`
@ -33,7 +33,7 @@ type KisFuncConfig struct {
connConf *KisConnConfig connConf *KisConnConfig
} }
// NewFuncConfig 创建一个Function策略配置对象, 用于描述一个KisFunction信息 // NewFuncConfig creates a Function strategy configuration object, used to describe a KisFunction information
func NewFuncConfig( func NewFuncConfig(
funcName string, mode common.KisMode, funcName string, mode common.KisMode,
source *KisSource, option *KisFuncOption) *KisFuncConfig { source *KisSource, option *KisFuncOption) *KisFuncConfig {
@ -53,13 +53,13 @@ func NewFuncConfig(
config.FMode = string(mode) config.FMode = string(mode)
/* /*
// FunctionS 和 L 需要必传KisConnector参数,原因是S和L需要通过Connector进行建立流式关系 // Functions S and L require the KisConnector parameters to be passed as they need to establish streaming relationships through Connector
if mode == common.S || mode == common.L { if mode == common.S || mode == common.L {
if option == nil { if option == nil {
log.Logger().ErrorF("Funcion S/L need option->Cid\n") log.Logger().ErrorF("Function S/L needs option->Cid\n")
return nil return nil
} else if option.CName == "" { } else if option.CName == "" {
log.Logger().ErrorF("Funcion S/L need option->Cid\n") log.Logger().ErrorF("Function S/L needs option->Cid\n")
return nil return nil
} }
} }
@ -72,23 +72,25 @@ func NewFuncConfig(
return config return config
} }
// AddConnConfig WithConn binds Function to Connector
func (fConf *KisFuncConfig) AddConnConfig(cConf *KisConnConfig) error { func (fConf *KisFuncConfig) AddConnConfig(cConf *KisConnConfig) error {
if cConf == nil { if cConf == nil {
return errors.New("KisConnConfig is nil") return errors.New("KisConnConfig is nil")
} }
// Function需要和Connector进行关联 // Function needs to be associated with Connector
fConf.connConf = cConf fConf.connConf = cConf
// Connector需要和Function进行关联 // Connector needs to be associated with Function
_ = cConf.WithFunc(fConf) _ = cConf.WithFunc(fConf)
// 更新Function配置中的CName // Update CName in Function configuration
fConf.Option.CName = cConf.CName fConf.Option.CName = cConf.CName
return nil return nil
} }
// GetConnConfig gets the Connector configuration
func (fConf *KisFuncConfig) GetConnConfig() (*KisConnConfig, error) { func (fConf *KisFuncConfig) GetConnConfig() (*KisConnConfig, error) {
if fConf.connConf == nil { if fConf.connConf == nil {
return nil, errors.New("KisFuncConfig.connConf not set") return nil, errors.New("KisFuncConfig.connConf not set")

View File

@ -1,15 +1,16 @@
package config package config
// KisGlobalConfig represents the global configuration for KisFlow
type KisGlobalConfig struct { type KisGlobalConfig struct {
//kistype Global为kisflow的全局配置 // KisType Global is the global configuration for kisflow
KisType string `yaml:"kistype"` KisType string `yaml:"kistype"`
//是否启动prometheus监控 // EnableProm indicates whether to start Prometheus monitoring
EnableProm bool `yaml:"prometheus_enable"` EnableProm bool `yaml:"prometheus_enable"`
//是否需要kisflow单独启动端口监听 // PrometheusListen indicates whether kisflow needs to start a separate port for listening
PrometheusListen bool `yaml:"prometheus_listen"` PrometheusListen bool `yaml:"prometheus_listen"`
//prometheus取点监听地址 // PrometheusServe is the address for Prometheus scraping
PrometheusServe string `yaml:"prometheus_serve"` PrometheusServe string `yaml:"prometheus_serve"`
} }
// GlobalConfig 默认全局配置,全部均为关闭 // GlobalConfig is the default global configuration, all are set to off
var GlobalConfig = new(KisGlobalConfig) var GlobalConfig = new(KisGlobalConfig)

View File

@ -9,6 +9,7 @@ import (
"sync" "sync"
) )
// KisConnector represents a KisConnector instance
type KisConnector struct { type KisConnector struct {
// Connector ID // Connector ID
CId string CId string
@ -20,16 +21,16 @@ type KisConnector struct {
// Connector Init // Connector Init
onceInit sync.Once onceInit sync.Once
// KisConnector的自定义临时数据 // KisConnector's custom temporary data
metaData map[string]interface{} metaData map[string]interface{}
// 管理metaData的读写锁 // Lock for reading and writing metaData
mLock sync.RWMutex mLock sync.RWMutex
} }
// NewKisConnector 根据配置策略创建一个KisConnector // NewKisConnector creates a KisConnector based on the given configuration
func NewKisConnector(config *config.KisConnConfig) *KisConnector { func NewKisConnector(config *config.KisConnConfig) *KisConnector {
conn := new(KisConnector) conn := new(KisConnector)
conn.CId = id.KisID(common.KisIdTypeConnector) conn.CId = id.KisID(common.KisIDTypeConnector)
conn.CName = config.CName conn.CName = config.CName
conn.Conf = config conn.Conf = config
conn.metaData = make(map[string]interface{}) conn.metaData = make(map[string]interface{})
@ -37,11 +38,11 @@ func NewKisConnector(config *config.KisConnConfig) *KisConnector {
return conn return conn
} }
// Init 初始化Connector所关联的存储引擎链接等 // Init initializes the connection to the associated storage engine of the Connector
func (conn *KisConnector) Init() error { func (conn *KisConnector) Init() error {
var err error var err error
// 一个Connector只能执行初始化业务一次 // The initialization business of a Connector can only be executed once
conn.onceInit.Do(func() { conn.onceInit.Do(func() {
err = kis.Pool().CallConnInit(conn) err = kis.Pool().CallConnInit(conn)
}) })
@ -49,7 +50,7 @@ func (conn *KisConnector) Init() error {
return err return err
} }
// Call 调用Connector 外挂存储逻辑的读写操作 // Call invokes the read-write operations of the external storage logic through the Connector
func (conn *KisConnector) Call(ctx context.Context, flow kis.Flow, args interface{}) (interface{}, error) { func (conn *KisConnector) Call(ctx context.Context, flow kis.Flow, args interface{}) (interface{}, error) {
var result interface{} var result interface{}
var err error var err error
@ -62,19 +63,22 @@ func (conn *KisConnector) Call(ctx context.Context, flow kis.Flow, args interfac
return result, nil return result, nil
} }
// GetName returns the name of the Connector
func (conn *KisConnector) GetName() string { func (conn *KisConnector) GetName() string {
return conn.CName return conn.CName
} }
// GetConfig returns the configuration of the Connector
func (conn *KisConnector) GetConfig() *config.KisConnConfig { func (conn *KisConnector) GetConfig() *config.KisConnConfig {
return conn.Conf return conn.Conf
} }
func (conn *KisConnector) GetId() string { // GetID returns the ID of the Connector
func (conn *KisConnector) GetID() string {
return conn.CId return conn.CId
} }
// GetMetaData 得到当前Connector的临时数据 // GetMetaData gets the temporary data of the current Connector
func (conn *KisConnector) GetMetaData(key string) interface{} { func (conn *KisConnector) GetMetaData(key string) interface{} {
conn.mLock.RLock() conn.mLock.RLock()
defer conn.mLock.RUnlock() defer conn.mLock.RUnlock()
@ -87,7 +91,7 @@ func (conn *KisConnector) GetMetaData(key string) interface{} {
return data return data
} }
// SetMetaData 设置当前Connector的临时数据 // SetMetaData sets the temporary data of the current Connector
func (conn *KisConnector) SetMetaData(key string, value interface{}) { func (conn *KisConnector) SetMetaData(key string, value interface{}) {
conn.mLock.Lock() conn.mLock.Lock()
defer conn.mLock.Unlock() defer conn.mLock.Unlock()

View File

@ -1,7 +1,6 @@
package file package file
import ( import (
"errors"
"fmt" "fmt"
"github.com/aceld/kis-flow/common" "github.com/aceld/kis-flow/common"
"github.com/aceld/kis-flow/kis" "github.com/aceld/kis-flow/kis"
@ -10,46 +9,53 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
// ConfigExportYaml 将flow配置输出且存储本地 // ConfigExportYaml exports the flow configuration and saves it locally
func ConfigExportYaml(flow kis.Flow, savaPath string) error { func ConfigExportYaml(flow kis.Flow, savePath string) error {
if data, err := yaml.Marshal(flow.GetConfig()); err != nil { var data []byte
var err error
data, err = yaml.Marshal(flow.GetConfig())
if err != nil {
return err return err
} else { }
// flow
err := os.WriteFile(savaPath+common.KisIdTypeFlow+"-"+flow.GetName()+".yaml", data, 0644) // flow
err = os.WriteFile(savePath+common.KisIDTypeFlow+"-"+flow.GetName()+".yaml", data, 0644)
if err != nil {
return err
}
// function
for _, fp := range flow.GetConfig().Flows {
fConf := flow.GetFuncConfigByName(fp.FuncName)
if fConf == nil {
return fmt.Errorf("function name = %s config is nil ", fp.FuncName)
}
fData, err := yaml.Marshal(fConf)
if err != nil { if err != nil {
return err return err
} }
// function if err := os.WriteFile(savePath+common.KisIDTypeFunction+"-"+fp.FuncName+".yaml", fData, 0644); err != nil {
for _, fp := range flow.GetConfig().Flows { return err
fConf := flow.GetFuncConfigByName(fp.FuncName) }
if fConf == nil {
return errors.New(fmt.Sprintf("function name = %s config is nil ", fp.FuncName))
}
if fdata, err := yaml.Marshal(fConf); err != nil { // Connector
if fConf.Option.CName != "" {
cConf, err := fConf.GetConnConfig()
if err != nil {
return err return err
} else {
if err := os.WriteFile(savaPath+common.KisIdTypeFunction+"-"+fp.FuncName+".yaml", fdata, 0644); err != nil {
return err
}
} }
// Connector cdata, err := yaml.Marshal(cConf)
if fConf.Option.CName != "" { if err != nil {
cConf, err := fConf.GetConnConfig() return err
if err != nil { }
return err
} if err := os.WriteFile(savePath+common.KisIDTypeConnector+"-"+cConf.CName+".yaml", cdata, 0644); err != nil {
if cdata, err := yaml.Marshal(cConf); err != nil { return err
return err
} else {
if err := os.WriteFile(savaPath+common.KisIdTypeConnector+"-"+cConf.CName+".yaml", cdata, 0644); err != nil {
return err
}
}
} }
} }
} }

View File

@ -1,18 +1,16 @@
package file package file
import ( import (
"errors"
"fmt" "fmt"
"github.com/aceld/kis-flow/common" "github.com/aceld/kis-flow/common"
"github.com/aceld/kis-flow/config" "github.com/aceld/kis-flow/config"
"github.com/aceld/kis-flow/flow" "github.com/aceld/kis-flow/flow"
"github.com/aceld/kis-flow/kis" "github.com/aceld/kis-flow/kis"
"github.com/aceld/kis-flow/metrics" "github.com/aceld/kis-flow/metrics"
"gopkg.in/yaml.v3"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"gopkg.in/yaml.v3"
) )
type allConfig struct { type allConfig struct {
@ -21,75 +19,75 @@ type allConfig struct {
Conns map[string]*config.KisConnConfig Conns map[string]*config.KisConnConfig
} }
// kisTypeFlowConfigure 解析Flow配置文件yaml格式 // kisTypeFlowConfigure parses Flow configuration file in yaml format
func kisTypeFlowConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { func kisTypeFlowConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error {
flowCfg := new(config.KisFlowConfig) flowCfg := new(config.KisFlowConfig)
if ok := yaml.Unmarshal(confData, flowCfg); ok != nil { if ok := yaml.Unmarshal(confData, flowCfg); ok != nil {
return errors.New(fmt.Sprintf("%s has wrong format kisType = %s", fileName, kisType)) return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType)
} }
// 如果FLow状态为关闭则不做配置加载 // Skip the configuration loading if the Flow status is disabled
if common.KisOnOff(flowCfg.Status) == common.FlowDisable { if common.KisOnOff(flowCfg.Status) == common.FlowDisable {
return nil return nil
} }
if _, ok := all.Flows[flowCfg.FlowName]; ok { if _, ok := all.Flows[flowCfg.FlowName]; ok {
return errors.New(fmt.Sprintf("%s set repeat flow_id:%s", fileName, flowCfg.FlowName)) return fmt.Errorf("%s set repeat flow_id:%s", fileName, flowCfg.FlowName)
} }
// 加入配置集合中 // Add to the configuration set
all.Flows[flowCfg.FlowName] = flowCfg all.Flows[flowCfg.FlowName] = flowCfg
return nil return nil
} }
// kisTypeFuncConfigure 解析Function配置文件yaml格式 // kisTypeFuncConfigure parses Function configuration file in yaml format
func kisTypeFuncConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { func kisTypeFuncConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error {
function := new(config.KisFuncConfig) function := new(config.KisFuncConfig)
if ok := yaml.Unmarshal(confData, function); ok != nil { if ok := yaml.Unmarshal(confData, function); ok != nil {
return errors.New(fmt.Sprintf("%s has wrong format kisType = %s", fileName, kisType)) return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType)
} }
if _, ok := all.Funcs[function.FName]; ok { if _, ok := all.Funcs[function.FName]; ok {
return errors.New(fmt.Sprintf("%s set repeat function_id:%s", fileName, function.FName)) return fmt.Errorf("%s set repeat function_id:%s", fileName, function.FName)
} }
// 加入配置集合中 // Add to the configuration set
all.Funcs[function.FName] = function all.Funcs[function.FName] = function
return nil return nil
} }
// kisTypeConnConfigure 解析Connector配置文件yaml格式 // kisTypeConnConfigure parses Connector configuration file in yaml format
func kisTypeConnConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { func kisTypeConnConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error {
conn := new(config.KisConnConfig) conn := new(config.KisConnConfig)
if ok := yaml.Unmarshal(confData, conn); ok != nil { if ok := yaml.Unmarshal(confData, conn); ok != nil {
return errors.New(fmt.Sprintf("%s is wrong format kisType = %s", fileName, kisType)) return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType)
} }
if _, ok := all.Conns[conn.CName]; ok { if _, ok := all.Conns[conn.CName]; ok {
return errors.New(fmt.Sprintf("%s set repeat conn_id:%s", fileName, conn.CName)) return fmt.Errorf("%s set repeat conn_id:%s", fileName, conn.CName)
} }
// 加入配置集合中 // Add to the configuration set
all.Conns[conn.CName] = conn all.Conns[conn.CName] = conn
return nil return nil
} }
// kisTypeGlobalConfigure 解析Global配置文件yaml格式 // kisTypeGlobalConfigure parses Global configuration file in yaml format
func kisTypeGlobalConfigure(confData []byte, fileName string, kisType interface{}) error { func kisTypeGlobalConfigure(confData []byte, fileName string, kisType interface{}) error {
// 全局配置 // Global configuration
if ok := yaml.Unmarshal(confData, config.GlobalConfig); ok != nil { if ok := yaml.Unmarshal(confData, config.GlobalConfig); ok != nil {
return errors.New(fmt.Sprintf("%s is wrong format kisType = %s", fileName, kisType)) return fmt.Errorf("%s is wrong format kisType = %s", fileName, kisType)
} }
// 启动Metrics服务 // Start Metrics service
metrics.RunMetrics() metrics.RunMetrics()
return nil return nil
} }
// parseConfigWalkYaml 全盘解析配置文件yaml格式, 讲配置信息解析到allConfig中 // parseConfigWalkYaml recursively parses all configuration files in yaml format and stores the configuration information in allConfig
func parseConfigWalkYaml(loadPath string) (*allConfig, error) { func parseConfigWalkYaml(loadPath string) (*allConfig, error) {
all := new(allConfig) all := new(allConfig)
@ -99,12 +97,12 @@ func parseConfigWalkYaml(loadPath string) (*allConfig, error) {
all.Conns = make(map[string]*config.KisConnConfig) all.Conns = make(map[string]*config.KisConnConfig)
err := filepath.Walk(loadPath, func(filePath string, info os.FileInfo, err error) error { err := filepath.Walk(loadPath, func(filePath string, info os.FileInfo, err error) error {
// 校验文件后缀是否合法 // Validate the file extension
if suffix := path.Ext(filePath); suffix != ".yml" && suffix != ".yaml" { if suffix := path.Ext(filePath); suffix != ".yml" && suffix != ".yaml" {
return nil return nil
} }
// 读取文件内容 // Read file content
confData, err := os.ReadFile(filePath) confData, err := os.ReadFile(filePath)
if err != nil { if err != nil {
return err return err
@ -112,31 +110,34 @@ func parseConfigWalkYaml(loadPath string) (*allConfig, error) {
confMap := make(map[string]interface{}) confMap := make(map[string]interface{})
// 校验yaml合法性 // Validate yaml format
if err := yaml.Unmarshal(confData, confMap); err != nil { if err := yaml.Unmarshal(confData, confMap); err != nil {
return err return err
} }
// 判断kisType是否存在 // Check if kisType exists
if kisType, ok := confMap["kistype"]; !ok { var kisType interface{}
return errors.New(fmt.Sprintf("yaml file %s has no file [kistype]!", filePath))
} else {
switch kisType {
case common.KisIdTypeFlow:
return kisTypeFlowConfigure(all, confData, filePath, kisType)
case common.KisIdTypeFunction: kisType, ok := confMap["kistype"]
return kisTypeFuncConfigure(all, confData, filePath, kisType) if !ok {
return fmt.Errorf("%s has no field [kistype]", filePath)
}
case common.KisIdTypeConnector: switch kisType {
return kisTypeConnConfigure(all, confData, filePath, kisType) case common.KisIDTypeFlow:
return kisTypeFlowConfigure(all, confData, filePath, kisType)
case common.KisIdTypeGlobal: case common.KisIDTypeFunction:
return kisTypeGlobalConfigure(confData, filePath, kisType) return kisTypeFuncConfigure(all, confData, filePath, kisType)
default: case common.KisIDTypeConnector:
return errors.New(fmt.Sprintf("%s set wrong kistype %s", filePath, kisType)) return kisTypeConnConfigure(all, confData, filePath, kisType)
}
case common.KisIDTypeGlobal:
return kisTypeGlobalConfigure(confData, filePath, kisType)
default:
return fmt.Errorf("%s set wrong kistype %s", filePath, kisType)
} }
}) })
@ -148,17 +149,17 @@ func parseConfigWalkYaml(loadPath string) (*allConfig, error) {
} }
func buildFlow(all *allConfig, fp config.KisFlowFunctionParam, newFlow kis.Flow, flowName string) error { func buildFlow(all *allConfig, fp config.KisFlowFunctionParam, newFlow kis.Flow, flowName string) error {
// 加载当前Flow依赖的Function // Load the Functions that the current Flow depends on
if funcConfig, ok := all.Funcs[fp.FuncName]; !ok { if funcConfig, ok := all.Funcs[fp.FuncName]; !ok {
return errors.New(fmt.Sprintf("FlowName [%s] need FuncName [%s], But has No This FuncName Config", flowName, fp.FuncName)) return fmt.Errorf("FlowName [%s] need FuncName [%s], But has No This FuncName Config", flowName, fp.FuncName)
} else { } else {
// flow add connector // flow add connector
if funcConfig.Option.CName != "" { if funcConfig.Option.CName != "" {
// 加载当前Function依赖的Connector // Load the Connectors that the current Function depends on
if connConf, ok := all.Conns[funcConfig.Option.CName]; !ok { if connConf, ok := all.Conns[funcConfig.Option.CName]; !ok {
return errors.New(fmt.Sprintf("FuncName [%s] need ConnName [%s], But has No This ConnName Config", fp.FuncName, funcConfig.Option.CName)) return fmt.Errorf("FuncName [%s] need ConnName [%s], But has No This ConnName Config", fp.FuncName, funcConfig.Option.CName)
} else { } else {
// Function Config 关联 Connector Config // Function Config associates with Connector Config
_ = funcConfig.AddConnConfig(connConf) _ = funcConfig.AddConnConfig(connConf)
} }
} }
@ -172,7 +173,7 @@ func buildFlow(all *allConfig, fp config.KisFlowFunctionParam, newFlow kis.Flow,
return nil return nil
} }
// ConfigImportYaml 全盘解析配置文件yaml格式 // ConfigImportYaml recursively parses all configuration files in yaml format
func ConfigImportYaml(loadPath string) error { func ConfigImportYaml(loadPath string) error {
all, err := parseConfigWalkYaml(loadPath) all, err := parseConfigWalkYaml(loadPath)
@ -182,7 +183,7 @@ func ConfigImportYaml(loadPath string) error {
for flowName, flowConfig := range all.Flows { for flowName, flowConfig := range all.Flows {
// 构建一个Flow // Build a new Flow
newFlow := flow.NewKisFlow(flowConfig) newFlow := flow.NewKisFlow(flowConfig)
for _, fp := range flowConfig.Flows { for _, fp := range flowConfig.Flows {
@ -191,7 +192,7 @@ func ConfigImportYaml(loadPath string) error {
} }
} }
// 将flow添加到FlowPool中 // Add the flow to FlowPool
kis.Pool().AddFlow(flowName, newFlow) kis.Pool().AddFlow(flowName, newFlow)
} }

View File

@ -18,81 +18,81 @@ import (
"github.com/patrickmn/go-cache" "github.com/patrickmn/go-cache"
) )
// KisFlow 用于贯穿整条流式计算的上下文环境 // KisFlow is used to manage the context environment of the entire streaming computation.
type KisFlow struct { type KisFlow struct {
// 基础信息 // Basic information
Id string // Flow的分布式实例ID(用于KisFlow内部区分不同实例) Id string // Distributed instance ID of the Flow (used internally by KisFlow to distinguish different instances)
Name string // Flow的可读名称 Name string // Readable name of the Flow
Conf *config.KisFlowConfig // Flow配置策略 Conf *config.KisFlowConfig // Flow configuration policy
// Function列表 // List of Functions
Funcs map[string]kis.Function // 当前flow拥有的全部管理的全部Function对象, key: FunctionName Funcs map[string]kis.Function // All managed Function objects of the current flow, key: FunctionName
FlowHead kis.Function // 当前Flow所拥有的Function列表表头 FlowHead kis.Function // Head of the Function list owned by the current Flow
FlowTail kis.Function // 当前Flow所拥有的Function列表表尾 FlowTail kis.Function // Tail of the Function list owned by the current Flow
flock sync.RWMutex // 管理链表插入读写的锁 flock sync.RWMutex // Lock for managing linked list insertion and reading
ThisFunction kis.Function // Flow当前正在执行的KisFunction对象 ThisFunction kis.Function // KisFunction object currently being executed in the Flow
ThisFunctionId string // 当前执行到的Function ID ThisFunctionId string // ID of the Function currently being executed
PrevFunctionId string // 当前执行到的Function 上一层FunctionID PrevFunctionId string // ID of the previous layer Function
// Function列表参数 // Function list parameters
funcParams map[string]config.FParam // flow在当前Function的自定义固定配置参数,Key:function的实例KisID, value:FParam funcParams map[string]config.FParam // Custom fixed configuration parameters of the Flow in the current Function, Key: KisID of the function instance, value: FParam
fplock sync.RWMutex // 管理funcParams的读写锁 fplock sync.RWMutex // Lock for managing funcParams read and write
// 数据 // Data
buffer common.KisRowArr // 用来临时存放输入字节数据的内部Buf, 一条数据为interface{}, 多条数据为[]interface{} 也就是KisBatch buffer common.KisRowArr // Internal buffer used to temporarily store input byte data, one data is interface{}, multiple data is []interface{} i.e. KisBatch
data common.KisDataMap // 流式计算各个层级的数据源 data common.KisDataMap // Data sources at various levels of the streaming computation
inPut common.KisRowArr // 当前Function的计算输入数据 inPut common.KisRowArr // Input data for the current Function computation
abort bool // 是否中断Flow abort bool // Whether to abort the Flow
action kis.Action // 当前Flow所携带的Action动作 action kis.Action // Action carried by the current Flow
// flow的本地缓存 // Local cache of the flow
cache *cache.Cache // Flow流的临时缓存上线文环境 cache *cache.Cache // Temporary cache context environment of the Flow
// flow的metaData // metaData of the flow
metaData map[string]interface{} // Flow的自定义临时数据 metaData map[string]interface{} // Custom temporary data of the Flow
mLock sync.RWMutex // 管理metaData的读写锁 mLock sync.RWMutex // Lock for managing metaData read and write
} }
// NewKisFlow 创建一个KisFlow. // NewKisFlow creates a KisFlow.
func NewKisFlow(conf *config.KisFlowConfig) kis.Flow { func NewKisFlow(conf *config.KisFlowConfig) kis.Flow {
flow := new(KisFlow) flow := new(KisFlow)
// 实例Id // Instance Id
flow.Id = id.KisID(common.KisIdTypeFlow) flow.Id = id.KisID(common.KisIDTypeFlow)
// 基础信息 // Basic information
flow.Name = conf.FlowName flow.Name = conf.FlowName
flow.Conf = conf flow.Conf = conf
// Function列表 // List of Functions
flow.Funcs = make(map[string]kis.Function) flow.Funcs = make(map[string]kis.Function)
flow.funcParams = make(map[string]config.FParam) flow.funcParams = make(map[string]config.FParam)
// 数据data // Data
flow.data = make(common.KisDataMap) flow.data = make(common.KisDataMap)
// 初始化本地缓存 // Initialize local cache
flow.cache = cache.New(cache.NoExpiration, common.DeFaultFlowCacheCleanUp*time.Minute) flow.cache = cache.New(cache.NoExpiration, common.DeFaultFlowCacheCleanUp*time.Minute)
// 初始化临时数据 // Initialize temporary data
flow.metaData = make(map[string]interface{}) flow.metaData = make(map[string]interface{})
return flow return flow
} }
// Fork 得到Flow的一个副本(深拷贝) // Fork gets a copy (deep copy) of the Flow.
func (flow *KisFlow) Fork(ctx context.Context) kis.Flow { func (flow *KisFlow) Fork(ctx context.Context) kis.Flow {
cfg := flow.Conf cfg := flow.Conf
// 通过之前的配置生成一个新的Flow // Generate a new Flow based on the previous configuration
newFlow := NewKisFlow(cfg) newFlow := NewKisFlow(cfg)
for _, fp := range flow.Conf.Flows { for _, fp := range flow.Conf.Flows {
if _, ok := flow.funcParams[flow.Funcs[fp.FuncName].GetId()]; !ok { if _, ok := flow.funcParams[flow.Funcs[fp.FuncName].GetID()]; !ok {
// 当前function没有配置Params // The current function has no Params configured
_ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), nil) _ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), nil)
} else { } else {
// 当前function有配置Params // The current function has configured Params
_ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), fp.Params) _ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), fp.Params)
} }
} }
@ -103,15 +103,15 @@ func (flow *KisFlow) Fork(ctx context.Context) kis.Flow {
return newFlow return newFlow
} }
// Link 将Function链接到Flow中, 同时会将Function的配置参数添加到Flow的配置中 // Link links the Function to the Flow, and also adds the Function's configuration parameters to the Flow's configuration.
// fConf: 当前Function策略 // fConf: Current Function strategy
// fParams: 当前Flow携带的Function动态参数 // fParams: Dynamic parameters carried by the current Flow's Function
func (flow *KisFlow) Link(fConf *config.KisFuncConfig, fParams config.FParam) error { func (flow *KisFlow) Link(fConf *config.KisFuncConfig, fParams config.FParam) error {
// Flow 添加Function // Add Function to Flow
_ = flow.AppendNewFunction(fConf, fParams) _ = flow.AppendNewFunction(fConf, fParams)
// FlowConfig 添加Function // Add Function to FlowConfig
flowFuncParam := config.KisFlowFunctionParam{ flowFuncParam := config.KisFlowFunctionParam{
FuncName: fConf.FName, FuncName: fConf.FName,
Params: fParams, Params: fParams,
@ -121,33 +121,33 @@ func (flow *KisFlow) Link(fConf *config.KisFuncConfig, fParams config.FParam) er
return nil return nil
} }
// AppendNewFunction 将一个新的Function追加到到Flow中 // AppendNewFunction appends a new Function to the Flow.
func (flow *KisFlow) AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error { func (flow *KisFlow) AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error {
// 创建Function实例 // Create Function instance
f := function.NewKisFunction(flow, fConf) f := function.NewKisFunction(flow, fConf)
if fConf.Option.CName != "" { if fConf.Option.CName != "" {
// 当前Function有Connector关联需要初始化Connector实例 // The current Function has a Connector association and needs to initialize the Connector instance
// 获取Connector配置 // Get Connector configuration
connConfig, err := fConf.GetConnConfig() connConfig, err := fConf.GetConnConfig()
if err != nil { if err != nil {
panic(err) panic(err)
} }
// 创建Connector对象 // Create Connector object
connector := conn.NewKisConnector(connConfig) connector := conn.NewKisConnector(connConfig)
// 初始化Connector, 执行Connector Init 方法 // Initialize Connector, execute the Connector Init method
if err = connector.Init(); err != nil { if err = connector.Init(); err != nil {
panic(err) panic(err)
} }
// 关联Function实例和Connector实例关系 // Associate the Function instance with the Connector instance
_ = f.AddConnector(connector) _ = f.AddConnector(connector)
} }
// Flow 添加 Function // Add Function to Flow
if err := flow.appendFunc(f, fParams); err != nil { if err := flow.appendFunc(f, fParams); err != nil {
return err return err
} }
@ -155,7 +155,7 @@ func (flow *KisFlow) AppendNewFunction(fConf *config.KisFuncConfig, fParams conf
return nil return nil
} }
// appendFunc 将Function添加到Flow中, 链表操作 // appendFunc adds the Function to the Flow, linked list operation
func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) error { func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) error {
if function == nil { if function == nil {
@ -166,7 +166,7 @@ func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) err
defer flow.flock.Unlock() defer flow.flock.Unlock()
if flow.FlowHead == nil { if flow.FlowHead == nil {
// 首次添加节点 // First time adding a node
flow.FlowHead = function flow.FlowHead = function
flow.FlowTail = function flow.FlowTail = function
@ -174,7 +174,7 @@ func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) err
function.SetP(nil) function.SetP(nil)
} else { } else {
// 将function插入到链表的尾部 // Insert the function at the end of the linked list
function.SetP(flow.FlowTail) function.SetP(flow.FlowTail)
function.SetN(nil) function.SetN(nil)
@ -182,28 +182,28 @@ func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) err
flow.FlowTail = function flow.FlowTail = function
} }
// 将Function Name 详细Hash对应关系添加到flow对象中 // Add the detailed Function Name-Hash correspondence to the flow object
flow.Funcs[function.GetConfig().FName] = function flow.Funcs[function.GetConfig().FName] = function
// 先添加function 默认携带的Params参数 // First add the Params parameters carried by the function by default
params := make(config.FParam) params := make(config.FParam)
for key, value := range function.GetConfig().Option.Params { for key, value := range function.GetConfig().Option.Params {
params[key] = value params[key] = value
} }
// 再添加flow携带的function定义参数(重复即覆盖) // Then add the function definition parameters carried by the flow (overwriting duplicates)
for key, value := range fParam { for key, value := range fParam {
params[key] = value params[key] = value
} }
// 将得到的FParams存留在flow结构体中用来function业务直接通过Hash获取 // Store the obtained FParams in the flow structure for direct access by the function
// key 为当前Function的KisId不用Fid的原因是为了防止一个Flow添加两个相同策略Id的Function // The key is the KisId of the current Function, not using Fid to prevent adding two Functions with the same strategy Id to a Flow
flow.funcParams[function.GetId()] = params flow.funcParams[function.GetID()] = params
return nil return nil
} }
// Run 启动KisFlow的流式计算, 从起始Function开始执行流 // Run starts the streaming computation of KisFlow, starting from the initial Function.
func (flow *KisFlow) Run(ctx context.Context) error { func (flow *KisFlow) Run(ctx context.Context) error {
var fn kis.Function var fn kis.Function
@ -212,7 +212,7 @@ func (flow *KisFlow) Run(ctx context.Context) error {
flow.abort = false flow.abort = false
if flow.Conf.Status == int(common.FlowDisable) { if flow.Conf.Status == int(common.FlowDisable) {
// flow被配置关闭 // Flow is configured to be disabled
return nil return nil
} }
@ -220,27 +220,27 @@ func (flow *KisFlow) Run(ctx context.Context) error {
var funcStart time.Time var funcStart time.Time
var flowStart time.Time var flowStart time.Time
// 因为此时还没有执行任何Function, 所以PrevFunctionId为FirstVirtual 因为没有上一层Function // Since no Function has been executed at this time, PrevFunctionId is FirstVirtual because there is no previous layer Function
flow.PrevFunctionId = common.FunctionIdFirstVirtual flow.PrevFunctionId = common.FunctionIDFirstVirtual
// 提交数据流原始数据 // Commit the original data stream
if err := flow.commitSrcData(ctx); err != nil { if err := flow.commitSrcData(ctx); err != nil {
return err return err
} }
// Metrics // Metrics
if config.GlobalConfig.EnableProm == true { if config.GlobalConfig.EnableProm == true {
// 统计Flow的调度次数 // Count the number of Flow schedules
metrics.Metrics.FlowScheduleCntsToTal.WithLabelValues(flow.Name).Inc() metrics.Metrics.FlowScheduleCntsToTal.WithLabelValues(flow.Name).Inc()
// 统计Flow的执行消耗时长 // Count the execution time of Flow
flowStart = time.Now() flowStart = time.Now()
} }
// 流式链式调用 // Streaming chain call
for fn != nil && flow.abort == false { for fn != nil && flow.abort == false {
// flow记录当前执行到的Function 标记 // Record the current Function being executed by the flow
fid := fn.GetId() fid := fn.GetID()
flow.ThisFunction = fn flow.ThisFunction = fn
flow.ThisFunctionId = fid flow.ThisFunctionId = fid
@ -248,14 +248,14 @@ func (flow *KisFlow) Run(ctx context.Context) error {
fMode := fn.GetConfig().FMode fMode := fn.GetConfig().FMode
if config.GlobalConfig.EnableProm == true { if config.GlobalConfig.EnableProm == true {
// 统计Function调度次数 // Count the number of Function schedules
metrics.Metrics.FuncScheduleCntsTotal.WithLabelValues(fName, fMode).Inc() metrics.Metrics.FuncScheduleCntsTotal.WithLabelValues(fName, fMode).Inc()
// 统计Function 耗时 记录开始时间 // Count the time consumed by Function, record the start time
funcStart = time.Now() funcStart = time.Now()
} }
// 得到当前Function要处理与的源数据 // Get the source data that the current Function needs to process
if inputData, err := flow.getCurData(); err != nil { if inputData, err := flow.getCurData(); err != nil {
log.Logger().ErrorFX(ctx, "flow.Run(): getCurData err = %s\n", err.Error()) log.Logger().ErrorFX(ctx, "flow.Run(): getCurData err = %s\n", err.Error())
return err return err
@ -273,16 +273,16 @@ func (flow *KisFlow) Run(ctx context.Context) error {
return err return err
} }
// 统计Function 耗时 // Count the time consumed by Function
if config.GlobalConfig.EnableProm == true { if config.GlobalConfig.EnableProm == true {
// Function消耗时间 // Function consumption time
duration := time.Since(funcStart) duration := time.Since(funcStart)
// 统计当前Function统计指标,做时间统计 // Count the current Function metrics, do time statistics
metrics.Metrics.FunctionDuration.With( metrics.Metrics.FunctionDuration.With(
prometheus.Labels{ prometheus.Labels{
common.LABEL_FUNCTION_NAME: fName, common.LabelFunctionName: fName,
common.LABEL_FUNCTION_MODE: fMode}).Observe(duration.Seconds() * 1000) common.LabelFunctionMode: fMode}).Observe(duration.Seconds() * 1000)
} }
} }
@ -290,7 +290,7 @@ func (flow *KisFlow) Run(ctx context.Context) error {
// Metrics // Metrics
if config.GlobalConfig.EnableProm == true { if config.GlobalConfig.EnableProm == true {
// 统计Flow执行耗时 // Count the execution time of Flow
duration := time.Since(flowStart) duration := time.Since(flowStart)
metrics.Metrics.FlowDuration.WithLabelValues(flow.Name).Observe(duration.Seconds() * 1000) metrics.Metrics.FlowDuration.WithLabelValues(flow.Name).Observe(duration.Seconds() * 1000)
} }
@ -298,10 +298,10 @@ func (flow *KisFlow) Run(ctx context.Context) error {
return nil return nil
} }
// Next 当前Flow执行到的Function进入下一层Function所携带的Action动作 // Next the current Flow enters the Action action carried by the next layer Function.
func (flow *KisFlow) Next(acts ...kis.ActionFunc) error { func (flow *KisFlow) Next(acts ...kis.ActionFunc) error {
// 加载Function FaaS 传递的 Action动作 // Load the Action actions carried by Function FaaS
flow.action = kis.LoadActions(acts) flow.action = kis.LoadActions(acts)
return nil return nil
@ -311,7 +311,7 @@ func (flow *KisFlow) GetName() string {
return flow.Name return flow.Name
} }
func (flow *KisFlow) GetId() string { func (flow *KisFlow) GetID() string {
return flow.Id return flow.Id
} }
@ -323,7 +323,7 @@ func (flow *KisFlow) GetThisFuncConf() *config.KisFuncConfig {
return flow.ThisFunction.GetConfig() return flow.ThisFunction.GetConfig()
} }
// GetConnector 得到当前正在执行的Function的Connector // GetConnector gets the Connector of the Function currently being executed by the Flow.
func (flow *KisFlow) GetConnector() (kis.Connector, error) { func (flow *KisFlow) GetConnector() (kis.Connector, error) {
if connector := flow.ThisFunction.GetConnector(); connector != nil { if connector := flow.ThisFunction.GetConnector(); connector != nil {
return connector, nil return connector, nil
@ -332,7 +332,7 @@ func (flow *KisFlow) GetConnector() (kis.Connector, error) {
} }
} }
// GetConnConf 得到当前正在执行的Function的Connector的配置 // GetConnConf gets the Connector configuration of the Function currently being executed by the Flow.
func (flow *KisFlow) GetConnConf() (*config.KisConnConfig, error) { func (flow *KisFlow) GetConnConf() (*config.KisConnConfig, error) {
if connector := flow.ThisFunction.GetConnector(); connector != nil { if connector := flow.ThisFunction.GetConnector(); connector != nil {
return connector.GetConfig(), nil return connector.GetConfig(), nil
@ -345,7 +345,7 @@ func (flow *KisFlow) GetConfig() *config.KisFlowConfig {
return flow.Conf return flow.Conf
} }
// GetFuncConfigByName 得到当前Flow的配置 // GetFuncConfigByName gets the configuration of the current Flow by Function name.
func (flow *KisFlow) GetFuncConfigByName(funcName string) *config.KisFuncConfig { func (flow *KisFlow) GetFuncConfigByName(funcName string) *config.KisFuncConfig {
if f, ok := flow.Funcs[funcName]; ok { if f, ok := flow.Funcs[funcName]; ok {
return f.GetConfig() return f.GetConfig()

View File

@ -7,7 +7,7 @@ import (
"github.com/aceld/kis-flow/kis" "github.com/aceld/kis-flow/kis"
) )
// dealAction 处理Action决定接下来Flow的流程走向 // dealAction handles Action to determine the next direction of the Flow.
func (flow *KisFlow) dealAction(ctx context.Context, fn kis.Function) (kis.Function, error) { func (flow *KisFlow) dealAction(ctx context.Context, fn kis.Function) (kis.Function, error) {
// DataReuse Action // DataReuse Action
@ -32,31 +32,31 @@ func (flow *KisFlow) dealAction(ctx context.Context, fn kis.Function) (kis.Funct
// JumpFunc Action // JumpFunc Action
if flow.action.JumpFunc != "" { if flow.action.JumpFunc != "" {
if _, ok := flow.Funcs[flow.action.JumpFunc]; !ok { if _, ok := flow.Funcs[flow.action.JumpFunc]; !ok {
//当前JumpFunc不在flow中 // The current JumpFunc is not in the flow
return nil, errors.New(fmt.Sprintf("Flow Jump -> %s is not in Flow", flow.action.JumpFunc)) return nil, errors.New(fmt.Sprintf("Flow Jump -> %s is not in Flow", flow.action.JumpFunc))
} }
jumpFunction := flow.Funcs[flow.action.JumpFunc] jumpFunction := flow.Funcs[flow.action.JumpFunc]
// 更新上层Function // Update the upper layer Function
flow.PrevFunctionId = jumpFunction.GetPrevId() flow.PrevFunctionId = jumpFunction.GetPrevId()
fn = jumpFunction fn = jumpFunction
// 如果设置跳跃,强制跳跃 // If set to jump, force the jump
flow.abort = false flow.abort = false
} else { } else {
// 更新上一层 FuncitonId 游标 // Update the upper layer FunctionId cursor
flow.PrevFunctionId = flow.ThisFunctionId flow.PrevFunctionId = flow.ThisFunctionId
fn = fn.Next() fn = fn.Next()
} }
// Abort Action 强制终止 // Abort Action force termination
if flow.action.Abort { if flow.action.Abort {
flow.abort = true flow.abort = true
} }
// 清空Action // Clear Action
flow.action = kis.Action{} flow.action = kis.Action{}
return fn, nil return fn, nil

View File

@ -13,7 +13,7 @@ import (
"time" "time"
) )
// CommitRow 提交Flow数据, 一行数据,如果是批量数据可以提交多次 // CommitRow submits a single row of data to the Flow; multiple rows can be submitted multiple times
func (flow *KisFlow) CommitRow(row interface{}) error { func (flow *KisFlow) CommitRow(row interface{}) error {
flow.buffer = append(flow.buffer, row) flow.buffer = append(flow.buffer, row)
@ -21,7 +21,7 @@ func (flow *KisFlow) CommitRow(row interface{}) error {
return nil return nil
} }
// CommitRowBatch 提交Flow数据, 批量数据 // CommitRowBatch submits a batch of data to the Flow
func (flow *KisFlow) CommitRowBatch(rows interface{}) error { func (flow *KisFlow) CommitRowBatch(rows interface{}) error {
v := reflect.ValueOf(rows) v := reflect.ValueOf(rows)
if v.Kind() != reflect.Slice { if v.Kind() != reflect.Slice {
@ -36,17 +36,17 @@ func (flow *KisFlow) CommitRowBatch(rows interface{}) error {
return nil return nil
} }
// Input 得到flow当前执行Function的输入源数据 // Input gets the input data for the currently executing Function in the Flow
func (flow *KisFlow) Input() common.KisRowArr { func (flow *KisFlow) Input() common.KisRowArr {
return flow.inPut return flow.inPut
} }
// commitSrcData 提交当前Flow的数据源数据, 表示首次提交当前Flow的原始数据源 // commitSrcData submits the data source data for the current Flow, indicating the first submission of the original data source for the current Flow
// 将flow的临时数据buffer提交到flow的data中,(data为各个Function层级的源数据备份) // The flow's temporary data buffer is submitted to the flow's data (data is the source data backup for each Function level)
// 会清空之前所有的flow数据 // All previous flow data will be cleared
func (flow *KisFlow) commitSrcData(ctx context.Context) error { func (flow *KisFlow) commitSrcData(ctx context.Context) error {
// 制作批量数据batch // Create a batch of data
dataCnt := len(flow.buffer) dataCnt := len(flow.buffer)
batch := make(common.KisRowArr, 0, dataCnt) batch := make(common.KisRowArr, 0, dataCnt)
@ -54,22 +54,22 @@ func (flow *KisFlow) commitSrcData(ctx context.Context) error {
batch = append(batch, row) batch = append(batch, row)
} }
// 清空之前所有数据 // Clear all previous data
flow.clearData(flow.data) flow.clearData(flow.data)
// 首次提交记录flow原始数据 // Record the original data for the flow for the first time
// 因为首次提交所以PrevFunctionId为FirstVirtual 因为没有上一层Function // Because it is the first submission, PrevFunctionId is FirstVirtual because there is no upper Function
flow.data[common.FunctionIdFirstVirtual] = batch flow.data[common.FunctionIDFirstVirtual] = batch
// 清空缓冲Buf // Clear the buffer
flow.buffer = flow.buffer[0:0] flow.buffer = flow.buffer[0:0]
// 首次提交数据源数据,进行统计数据总量 // The first submission of data source data, for statistical total data
if config.GlobalConfig.EnableProm == true { if config.GlobalConfig.EnableProm == true {
// 统计数据总量 Metrics.DataTota 指标累计加1 // Statistics for total data Metrics.DataTotal accumulates by 1
metrics.Metrics.DataTotal.Add(float64(dataCnt)) metrics.Metrics.DataTotal.Add(float64(dataCnt))
//统计当前Flow数量指标 // Statistics for current Flow quantity index
metrics.Metrics.FlowDataTotal.WithLabelValues(flow.Name).Add(float64(dataCnt)) metrics.Metrics.FlowDataTotal.WithLabelValues(flow.Name).Add(float64(dataCnt))
} }
@ -78,7 +78,7 @@ func (flow *KisFlow) commitSrcData(ctx context.Context) error {
return nil return nil
} }
// getCurData 获取flow当前Function层级的输入数据 // getCurData gets the input data for the current Function level of the flow
func (flow *KisFlow) getCurData() (common.KisRowArr, error) { func (flow *KisFlow) getCurData() (common.KisRowArr, error) {
if flow.PrevFunctionId == "" { if flow.PrevFunctionId == "" {
return nil, errors.New(fmt.Sprintf("flow.PrevFunctionId is not set")) return nil, errors.New(fmt.Sprintf("flow.PrevFunctionId is not set"))
@ -94,16 +94,16 @@ func (flow *KisFlow) getCurData() (common.KisRowArr, error) {
// commitReuseData // commitReuseData
func (flow *KisFlow) commitReuseData(ctx context.Context) error { func (flow *KisFlow) commitReuseData(ctx context.Context) error {
// 判断上层是否有结果数据, 如果没有则退出本次Flow Run循环 // Check if there are result data from the upper layer; if not, exit the current Flow Run loop
if len(flow.data[flow.PrevFunctionId]) == 0 { if len(flow.data[flow.PrevFunctionId]) == 0 {
flow.abort = true flow.abort = true
return nil return nil
} }
// 本层结果数据等于上层结果数据(复用上层结果数据到本层) // This layer's result data is equal to the upper layer's result data (reuse the upper layer's result data to this layer)
flow.data[flow.ThisFunctionId] = flow.data[flow.PrevFunctionId] flow.data[flow.ThisFunctionId] = flow.data[flow.PrevFunctionId]
// 清空缓冲Buf (如果是ReuseData选项那么提交的全部数据都将不会携带到下一层) // Clear the buffer (If it is a ReuseData option, all the submitted data will not be carried to the next layer)
flow.buffer = flow.buffer[0:0] flow.buffer = flow.buffer[0:0]
log.Logger().DebugFX(ctx, " ====> After commitReuseData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) log.Logger().DebugFX(ctx, " ====> After commitReuseData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data)
@ -116,10 +116,10 @@ func (flow *KisFlow) commitVoidData(ctx context.Context) error {
return nil return nil
} }
// 制作空数据 // Create empty data
batch := make(common.KisRowArr, 0) batch := make(common.KisRowArr, 0)
// 将本层计算的缓冲数据提交到本层结果数据中 // Submit the calculated buffer data of this layer to the result data of this layer
flow.data[flow.ThisFunctionId] = batch flow.data[flow.ThisFunctionId] = batch
log.Logger().DebugFX(ctx, " ====> After commitVoidData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) log.Logger().DebugFX(ctx, " ====> After commitVoidData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data)
@ -127,27 +127,27 @@ func (flow *KisFlow) commitVoidData(ctx context.Context) error {
return nil return nil
} }
//commitCurData 提交Flow当前执行Function的结果数据 // commitCurData submits the result data of the currently executing Function in the Flow
func (flow *KisFlow) commitCurData(ctx context.Context) error { func (flow *KisFlow) commitCurData(ctx context.Context) error {
// 判断本层计算是否有结果数据,如果没有则退出本次Flow Run循环 // Check if this layer's calculation has result data; if not, exit the current Flow Run loop
if len(flow.buffer) == 0 { if len(flow.buffer) == 0 {
flow.abort = true flow.abort = true
return nil return nil
} }
// 制作批量数据batch // Create a batch of data
batch := make(common.KisRowArr, 0, len(flow.buffer)) batch := make(common.KisRowArr, 0, len(flow.buffer))
// 如果strBuf为空则没有添加任何数据 // If strBuf is empty, no data has been added
for _, row := range flow.buffer { for _, row := range flow.buffer {
batch = append(batch, row) batch = append(batch, row)
} }
// 将本层计算的缓冲数据提交到本层结果数据中 // Submit the calculated buffer data of this layer to the result data of this layer
flow.data[flow.ThisFunctionId] = batch flow.data[flow.ThisFunctionId] = batch
// 清空缓冲Buf // Clear the buffer
flow.buffer = flow.buffer[0:0] flow.buffer = flow.buffer[0:0]
log.Logger().DebugFX(ctx, " ====> After commitCurData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) log.Logger().DebugFX(ctx, " ====> After commitCurData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data)
@ -155,7 +155,7 @@ func (flow *KisFlow) commitCurData(ctx context.Context) error {
return nil return nil
} }
// clearData 清空flow所有数据 // clearData clears all flow data
func (flow *KisFlow) clearData(data common.KisDataMap) { func (flow *KisFlow) clearData(data common.KisDataMap) {
for k := range data { for k := range data {
delete(data, k) delete(data, k)
@ -179,7 +179,7 @@ func (flow *KisFlow) SetCacheData(key string, value interface{}, Exp time.Durati
} }
} }
// GetMetaData 得到当前Flow对象的临时数据 // GetMetaData gets the temporary data of the current Flow object
func (flow *KisFlow) GetMetaData(key string) interface{} { func (flow *KisFlow) GetMetaData(key string) interface{} {
flow.mLock.RLock() flow.mLock.RLock()
defer flow.mLock.RUnlock() defer flow.mLock.RUnlock()
@ -192,7 +192,7 @@ func (flow *KisFlow) GetMetaData(key string) interface{} {
return data return data
} }
// SetMetaData 设置当前Flow对象的临时数据 // SetMetaData sets the temporary data of the current Flow object
func (flow *KisFlow) SetMetaData(key string, value interface{}) { func (flow *KisFlow) SetMetaData(key string, value interface{}) {
flow.mLock.Lock() flow.mLock.Lock()
defer flow.mLock.Unlock() defer flow.mLock.Unlock()
@ -200,7 +200,7 @@ func (flow *KisFlow) SetMetaData(key string, value interface{}) {
flow.metaData[key] = value flow.metaData[key] = value
} }
// GetFuncParam 得到Flow的当前正在执行的Function的配置默认参数取出一对key-value // GetFuncParam gets the default configuration parameters of the currently executing Function in the Flow, retrieves a key-value pair
func (flow *KisFlow) GetFuncParam(key string) string { func (flow *KisFlow) GetFuncParam(key string) string {
flow.fplock.RLock() flow.fplock.RLock()
defer flow.fplock.RUnlock() defer flow.fplock.RUnlock()
@ -214,7 +214,7 @@ func (flow *KisFlow) GetFuncParam(key string) string {
return "" return ""
} }
// GetFuncParamAll 得到Flow的当前正在执行的Function的配置默认参数取出全部Key-Value // GetFuncParamAll gets the default configuration parameters of the currently executing Function in the Flow, retrieves all Key-Value pairs
func (flow *KisFlow) GetFuncParamAll() config.FParam { func (flow *KisFlow) GetFuncParamAll() config.FParam {
flow.fplock.RLock() flow.fplock.RLock()
defer flow.fplock.RUnlock() defer flow.fplock.RUnlock()
@ -227,7 +227,7 @@ func (flow *KisFlow) GetFuncParamAll() config.FParam {
return param return param
} }
// GetFuncParamsAllFuncs 得到Flow中所有Function的FuncParams取出全部Key-Value // GetFuncParamsAllFuncs gets the FuncParams of all Functions in the Flow, retrieves all Key-Value pairs
func (flow *KisFlow) GetFuncParamsAllFuncs() map[string]config.FParam { func (flow *KisFlow) GetFuncParamsAllFuncs() map[string]config.FParam {
flow.fplock.RLock() flow.fplock.RLock()
defer flow.fplock.RUnlock() defer flow.fplock.RUnlock()

View File

@ -11,28 +11,29 @@ import (
) )
type BaseFunction struct { type BaseFunction struct {
// Id , KisFunction的实例ID用于KisFlow内部区分不同的实例对象 // Id, the instance ID of KisFunction, used to differentiate different instance objects within KisFlow
Id string Id string
Config *config.KisFuncConfig Config *config.KisFuncConfig
// flow // flow
flow kis.Flow //上下文环境KisFlow flow kis.Flow // Context environment KisFlow
// connector // connector
connector kis.Connector connector kis.Connector
// Function的自定义临时数据 // Custom temporary data of Function
metaData map[string]interface{} metaData map[string]interface{}
// 管理metaData的读写锁 // Manage the read-write lock of metaData
mLock sync.RWMutex mLock sync.RWMutex
// link // link
N kis.Function //下一个流计算Function N kis.Function // Next flow computing Function
P kis.Function //上一个流计算Function P kis.Function // Previous flow computing Function
} }
// Call // Call
// BaseFunction 为空实现目的为了让其他具体类型的KisFunction如KisFunction_V 来继承BaseFuncion来重写此方法 // BaseFunction is an empty implementation, designed to allow other specific types of KisFunction,
// such as KisFunction_V, to inherit BaseFuncion and override this method
func (base *BaseFunction) Call(ctx context.Context, flow kis.Flow) error { return nil } func (base *BaseFunction) Call(ctx context.Context, flow kis.Flow) error { return nil }
func (base *BaseFunction) Next() kis.Function { func (base *BaseFunction) Next() kis.Function {
@ -61,24 +62,24 @@ func (base *BaseFunction) SetConfig(s *config.KisFuncConfig) error {
return nil return nil
} }
func (base *BaseFunction) GetId() string { func (base *BaseFunction) GetID() string {
return base.Id return base.Id
} }
func (base *BaseFunction) GetPrevId() string { func (base *BaseFunction) GetPrevId() string {
if base.P == nil { if base.P == nil {
//Function为首结点 // Function is the first node
return common.FunctionIdFirstVirtual return common.FunctionIDFirstVirtual
} }
return base.P.GetId() return base.P.GetID()
} }
func (base *BaseFunction) GetNextId() string { func (base *BaseFunction) GetNextId() string {
if base.N == nil { if base.N == nil {
//Function为尾结点 // Function is the last node
return common.FunctionIdLastVirtual return common.FunctionIDLastVirtual
} }
return base.N.GetId() return base.N.GetID()
} }
func (base *BaseFunction) GetConfig() *config.KisFuncConfig { func (base *BaseFunction) GetConfig() *config.KisFuncConfig {
@ -97,7 +98,7 @@ func (base *BaseFunction) GetFlow() kis.Flow {
return base.flow return base.flow
} }
// AddConnector 给当前Function实例添加一个Connector // AddConnector adds a Connector to the current Function instance
func (base *BaseFunction) AddConnector(conn kis.Connector) error { func (base *BaseFunction) AddConnector(conn kis.Connector) error {
if conn == nil { if conn == nil {
return errors.New("conn is nil") return errors.New("conn is nil")
@ -108,22 +109,22 @@ func (base *BaseFunction) AddConnector(conn kis.Connector) error {
return nil return nil
} }
// GetConnector 获取当前Function实例所关联的Connector // GetConnector gets the Connector associated with the current Function instance
func (base *BaseFunction) GetConnector() kis.Connector { func (base *BaseFunction) GetConnector() kis.Connector {
return base.connector return base.connector
} }
func (base *BaseFunction) CreateId() { func (base *BaseFunction) CreateId() {
base.Id = id.KisID(common.KisIdTypeFunction) base.Id = id.KisID(common.KisIDTypeFunction)
} }
// NewKisFunction 创建一个NsFunction // NewKisFunction creates a new NsFunction
// flow: 当前所属的flow实例 // flow: the current belonging flow instance
// s : 当前function的配置策略 // s: the configuration strategy of the current function
func NewKisFunction(flow kis.Flow, config *config.KisFuncConfig) kis.Function { func NewKisFunction(flow kis.Flow, config *config.KisFuncConfig) kis.Function {
var f kis.Function var f kis.Function
//工厂生产泛化对象 // Factory produces generic objects
switch common.KisMode(config.FMode) { switch common.KisMode(config.FMode) {
case common.V: case common.V:
f = NewKisFunctionV() f = NewKisFunctionV()
@ -136,19 +137,19 @@ func NewKisFunction(flow kis.Flow, config *config.KisFuncConfig) kis.Function {
case common.E: case common.E:
f = NewKisFunctionE() f = NewKisFunctionE()
default: default:
//LOG ERROR // LOG ERROR
return nil return nil
} }
// 生成随机实例唯一ID // Generate a random unique instance ID
f.CreateId() f.CreateId()
// 设置基础信息属性 // Set basic information attributes
if err := f.SetConfig(config); err != nil { if err := f.SetConfig(config); err != nil {
panic(err) panic(err)
} }
// 设置Flow // Set Flow
if err := f.SetFlow(flow); err != nil { if err := f.SetFlow(flow); err != nil {
panic(err) panic(err)
} }
@ -156,7 +157,7 @@ func NewKisFunction(flow kis.Flow, config *config.KisFuncConfig) kis.Function {
return f return f
} }
// GetMetaData 得到当前Function的临时数据 // GetMetaData gets the temporary data of the current Function
func (base *BaseFunction) GetMetaData(key string) interface{} { func (base *BaseFunction) GetMetaData(key string) interface{} {
base.mLock.RLock() base.mLock.RLock()
defer base.mLock.RUnlock() defer base.mLock.RUnlock()
@ -169,7 +170,7 @@ func (base *BaseFunction) GetMetaData(key string) interface{} {
return data return data
} }
// SetMetaData 设置当前Function的临时数据 // SetMetaData sets the temporary data of the current Function
func (base *BaseFunction) SetMetaData(key string, value interface{}) { func (base *BaseFunction) SetMetaData(key string, value interface{}) {
base.mLock.Lock() base.mLock.Lock()
defer base.mLock.Unlock() defer base.mLock.Unlock()

View File

@ -13,7 +13,7 @@ type KisFunctionC struct {
func NewKisFunctionC() kis.Function { func NewKisFunctionC() kis.Function {
f := new(KisFunctionC) f := new(KisFunctionC)
// 初始化metaData // Initialize metaData
f.metaData = make(map[string]interface{}) f.metaData = make(map[string]interface{})
return f return f
@ -22,7 +22,7 @@ func NewKisFunctionC() kis.Function {
func (f *KisFunctionC) Call(ctx context.Context, flow kis.Flow) error { func (f *KisFunctionC) Call(ctx context.Context, flow kis.Flow) error {
log.Logger().DebugF("KisFunctionC, flow = %+v\n", flow) log.Logger().DebugF("KisFunctionC, flow = %+v\n", flow)
// 通过KisPool 路由到具体的执行计算Function中 // Route to the specific computing Function through KisPool
if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil {
log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err)
return err return err

View File

@ -13,7 +13,7 @@ type KisFunctionE struct {
func NewKisFunctionE() kis.Function { func NewKisFunctionE() kis.Function {
f := new(KisFunctionE) f := new(KisFunctionE)
// 初始化metaData // Initialize metaData
f.metaData = make(map[string]interface{}) f.metaData = make(map[string]interface{})
return f return f
@ -22,7 +22,7 @@ func NewKisFunctionE() kis.Function {
func (f *KisFunctionE) Call(ctx context.Context, flow kis.Flow) error { func (f *KisFunctionE) Call(ctx context.Context, flow kis.Flow) error {
log.Logger().DebugF("KisFunctionE, flow = %+v\n", flow) log.Logger().DebugF("KisFunctionE, flow = %+v\n", flow)
// 通过KisPool 路由到具体的执行计算Function中 // Route to the specific computing Function through KisPool
if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil {
log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err)
return err return err

View File

@ -13,7 +13,7 @@ type KisFunctionL struct {
func NewKisFunctionL() kis.Function { func NewKisFunctionL() kis.Function {
f := new(KisFunctionL) f := new(KisFunctionL)
// 初始化metaData // Initialize metaData
f.metaData = make(map[string]interface{}) f.metaData = make(map[string]interface{})
return f return f
@ -22,7 +22,7 @@ func NewKisFunctionL() kis.Function {
func (f *KisFunctionL) Call(ctx context.Context, flow kis.Flow) error { func (f *KisFunctionL) Call(ctx context.Context, flow kis.Flow) error {
log.Logger().DebugF("KisFunctionL, flow = %+v\n", flow) log.Logger().DebugF("KisFunctionL, flow = %+v\n", flow)
// 通过KisPool 路由到具体的执行计算Function中 // Route to the specific computing Function through KisPool
if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil {
log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err)
return err return err

View File

@ -13,7 +13,7 @@ type KisFunctionS struct {
func NewKisFunctionS() kis.Function { func NewKisFunctionS() kis.Function {
f := new(KisFunctionS) f := new(KisFunctionS)
// 初始化metaData // Initialize metaData
f.metaData = make(map[string]interface{}) f.metaData = make(map[string]interface{})
return f return f
@ -22,7 +22,7 @@ func NewKisFunctionS() kis.Function {
func (f *KisFunctionS) Call(ctx context.Context, flow kis.Flow) error { func (f *KisFunctionS) Call(ctx context.Context, flow kis.Flow) error {
log.Logger().DebugF("KisFunctionS, flow = %+v\n", flow) log.Logger().DebugF("KisFunctionS, flow = %+v\n", flow)
// 通过KisPool 路由到具体的执行计算Function中 // Route to the specific computing Function through KisPool
if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil {
log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err)
return err return err

View File

@ -13,7 +13,7 @@ type KisFunctionV struct {
func NewKisFunctionV() kis.Function { func NewKisFunctionV() kis.Function {
f := new(KisFunctionV) f := new(KisFunctionV)
// 初始化metaData // Initialize metaData
f.metaData = make(map[string]interface{}) f.metaData = make(map[string]interface{})
return f return f
@ -22,7 +22,7 @@ func NewKisFunctionV() kis.Function {
func (f *KisFunctionV) Call(ctx context.Context, flow kis.Flow) error { func (f *KisFunctionV) Call(ctx context.Context, flow kis.Flow) error {
log.Logger().DebugF("KisFunctionV, flow = %+v\n", flow) log.Logger().DebugF("KisFunctionV, flow = %+v\n", flow)
// 通过KisPool 路由到具体的执行计算Function中 // Route to the specific computing Function through KisPool
if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil {
log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err)
return err return err

2
go.mod
View File

@ -17,6 +17,6 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/sys v0.19.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
) )

View File

@ -6,12 +6,12 @@ import (
"strings" "strings"
) )
// KisID 获取一个中随机实例ID // KisID generates a random instance ID.
// 格式为 "prefix1-[prefix2-][prefix3-]ID" // The format is "prefix1-[prefix2-][prefix3-]ID"
// 如:flow-1234567890 // Example: flow-1234567890
// 如:func-1234567890 // Example: func-1234567890
// : conn-1234567890 // Example: conn-1234567890
// : func-1-1234567890 // Example: func-1-1234567890
func KisID(prefix ...string) (kisId string) { func KisID(prefix ...string) (kisId string) {
idStr := strings.Replace(uuid.New().String(), "-", "", -1) idStr := strings.Replace(uuid.New().String(), "-", "", -1)
@ -25,7 +25,7 @@ func formatKisID(idStr string, prefix ...string) string {
for _, fix := range prefix { for _, fix := range prefix {
kisId += fix kisId += fix
kisId += common.KisIdJoinChar kisId += common.KisIDJoinChar
} }
kisId += idStr kisId += idStr

View File

@ -1,25 +1,27 @@
package kis package kis
// Action KisFlow执行流程Actions // Action defines the actions for KisFlow execution.
type Action struct { type Action struct {
// DataReuse 是否复用上层Function数据 // DataReuse indicates whether to reuse data from the upper Function.
DataReuse bool DataReuse bool
// 默认Next()为如果本层Function计算结果为0条数据之后Function将不会继续执行 // ForceEntryNext overrides the default rule, where if the current Function's calculation result is 0 data entries,
// ForceEntryNext 为忽略上述默认规则没有数据强制进入下一层Function // subsequent Functions will not continue execution.
// With ForceEntryNext set to true, the next Function will be entered regardless of the data.
ForceEntryNext bool ForceEntryNext bool
// JumpFunc 跳转到指定Function继续执行 // JumpFunc specifies the Function to jump to for continued execution.
// (Note: This can easily lead to Flow loop calls, causing an infinite loop.)
JumpFunc string JumpFunc string
// Abort 终止Flow的执行 // Abort terminates the execution of the Flow.
Abort bool Abort bool
} }
// ActionFunc KisFlow Functional Option 类型 // ActionFunc is the type for KisFlow Functional Option.
type ActionFunc func(ops *Action) type ActionFunc func(ops *Action)
// LoadActions 加载Actions依次执行ActionFunc操作函数 // LoadActions loads Actions and sequentially executes the ActionFunc operations.
func LoadActions(acts []ActionFunc) Action { func LoadActions(acts []ActionFunc) Action {
action := Action{} action := Action{}
@ -34,25 +36,25 @@ func LoadActions(acts []ActionFunc) Action {
return action return action
} }
// ActionDataReuse Next复用上层Function数据Option // ActionDataReuse sets the option for reusing data from the upper Function.
func ActionDataReuse(act *Action) { func ActionDataReuse(act *Action) {
act.DataReuse = true act.DataReuse = true
} }
// ActionForceEntryNext 强制进入下一层 // ActionForceEntryNext sets the option to forcefully enter the next layer.
func ActionForceEntryNext(act *Action) { func ActionForceEntryNext(act *Action) {
act.ForceEntryNext = true act.ForceEntryNext = true
} }
// ActionJumpFunc 会返回一个ActionFunc函数并且会将funcName赋值给Action.JumpFunc // ActionJumpFunc returns an ActionFunc function and sets the funcName to Action.JumpFunc.
// (注意容易出现Flow循环调用导致死循环) // (Note: This can easily lead to Flow loop calls, causing an infinite loop.)
func ActionJumpFunc(funcName string) ActionFunc { func ActionJumpFunc(funcName string) ActionFunc {
return func(act *Action) { return func(act *Action) {
act.JumpFunc = funcName act.JumpFunc = funcName
} }
} }
// ActionAbort 终止Flow的执行 // ActionAbort terminates the execution of the Flow.
func ActionAbort(action *Action) { func ActionAbort(action *Action) {
action.Abort = true action.Abort = true
} }

View File

@ -5,19 +5,20 @@ import (
"github.com/aceld/kis-flow/config" "github.com/aceld/kis-flow/config"
) )
// Connector defines the interface for connectors associated with external storage.
type Connector interface { type Connector interface {
// Init 初始化Connector所关联的存储引擎链接等 // Init initializes the connection to the storage engine associated with the Connector.
Init() error Init() error
// Call 调用Connector 外挂存储逻辑的读写操作 // Call invokes the read-write operations of the external storage logic.
Call(ctx context.Context, flow Flow, args interface{}) (interface{}, error) Call(ctx context.Context, flow Flow, args interface{}) (interface{}, error)
// GetId 获取Connector的ID // GetID returns the ID of the Connector.
GetId() string GetID() string
// GetName 获取Connector的名称 // GetName returns the name of the Connector.
GetName() string GetName() string
// GetConfig 获取Connector的配置信息 // GetConfig returns the configuration information of the Connector.
GetConfig() *config.KisConnConfig GetConfig() *config.KisConnConfig
// GetMetaData 得到当前Connector的临时数据 // GetMetaData gets the temporary data of the current Connector.
GetMetaData(key string) interface{} GetMetaData(key string) interface{}
// SetMetaData 设置当前Connector的临时数据 // SetMetaData sets the temporary data of the current Connector.
SetMetaData(key string, value interface{}) SetMetaData(key string, value interface{})
} }

View File

@ -10,113 +10,111 @@ import (
// FaaS Function as a Service // FaaS Function as a Service
// // Change the type definition from:
// type FaaS func(context.Context, Flow) error // type FaaS func(context.Context, Flow) error
// 改为 // to:
// type FaaS func(context.Context, Flow, ...interface{}) error // type FaaS func(context.Context, Flow, ...interface{}) error
// 可以通过可变参数的任意输入类型进行数据传递 // This allows passing data through variadic parameters of any type.
type FaaS interface{} type FaaS interface{}
// FaaSDesc FaaS 回调计算业务函数 描述 // FaaSDesc describes the FaaS callback computation function.
type FaaSDesc struct { type FaaSDesc struct {
Serialize // 当前Function的数据输入输出序列化实现 Serialize // Serialization implementation for the current Function's data input and output
FnName string // Function名称 FnName string // Function name
f interface{} // FaaS 函数 f interface{} // FaaS function
fName string // 函数名称 fName string // Function name
ArgsType []reflect.Type // 函数参数类型(集合) ArgsType []reflect.Type // Function parameter types (collection)
ArgNum int // 函数参数个数 ArgNum int // Number of function parameters
FuncType reflect.Type // 函数类型 FuncType reflect.Type // Function type
FuncValue reflect.Value // 函数值(函数地址) FuncValue reflect.Value // Function value (function address)
} }
// NewFaaSDesc 根据用户注册的FnName 和FaaS 回调函数,创建 FaaSDesc 描述实例 // NewFaaSDesc creates an instance of FaaSDesc description based on the registered FnName and FaaS callback function.
func NewFaaSDesc(fnName string, f FaaS) (*FaaSDesc, error) { func NewFaaSDesc(fnName string, f FaaS) (*FaaSDesc, error) {
// 输入输出序列化实例 // Serialization instance
var serializeImpl Serialize var serializeImpl Serialize
// 传入的回调函数FaaS,函数值(函数地址) // Callback function value (function address)
funcValue := reflect.ValueOf(f) funcValue := reflect.ValueOf(f)
// 传入的回调函数FaaS 类型 // Callback function type
funcType := funcValue.Type() funcType := funcValue.Type()
// 判断传递的FaaS指针是否是函数类型 // Check if the provided FaaS pointer is a function type
if !isFuncType(funcType) { if !isFuncType(funcType) {
return nil, fmt.Errorf("provided FaaS type is %s, not a function", funcType.Name()) return nil, fmt.Errorf("provided FaaS type is %s, not a function", funcType.Name())
} }
// 判断传递的FaaS函数是否有返回值类型是只包括(error) // Check if the FaaS function has a return value that only includes (error)
if funcType.NumOut() != 1 || funcType.Out(0) != reflect.TypeOf((*error)(nil)).Elem() { if funcType.NumOut() != 1 || funcType.Out(0) != reflect.TypeOf((*error)(nil)).Elem() {
return nil, errors.New("function must have exactly one return value of type error") return nil, errors.New("function must have exactly one return value of type error")
} }
// FaaS函数的参数类型集合 // FaaS function parameter types
argsType := make([]reflect.Type, funcType.NumIn()) argsType := make([]reflect.Type, funcType.NumIn())
// 获取FaaS的函数名称 // Get the FaaS function name
fullName := runtime.FuncForPC(funcValue.Pointer()).Name() fullName := runtime.FuncForPC(funcValue.Pointer()).Name()
// 确保 FaaS func(context.Context, Flow, ...interface{}) error 形参列表存在context.Context 和 kis.Flow // Ensure that the FaaS function parameter list contains context.Context and kis.Flow
// Check if the function contains a parameter of type kis.Flow
// 是否包含kis.Flow类型的形参
containsKisFlow := false containsKisFlow := false
// 是否包含context.Context类型的形参 // Check if the function contains a parameter of type context.Context
containsCtx := false containsCtx := false
// 遍历FaaS的形参类型 // Iterate over the FaaS function parameter types
for i := 0; i < funcType.NumIn(); i++ { for i := 0; i < funcType.NumIn(); i++ {
// 取出第i个形式参数类型 // Get the i-th formal parameter type
paramType := funcType.In(i) paramType := funcType.In(i)
if isFlowType(paramType) { if isFlowType(paramType) {
// 判断是否包含kis.Flow类型的形参 // Check if the function contains a parameter of type kis.Flow
containsKisFlow = true containsKisFlow = true
} else if isContextType(paramType) { } else if isContextType(paramType) {
// 判断是否包含context.Context类型的形参 // Check if the function contains a parameter of type context.Context
containsCtx = true containsCtx = true
} else if isSliceType(paramType) { } else if isSliceType(paramType) {
// 获取当前参数Slice的元素类型 // Get the element type of the current parameter Slice
itemType := paramType.Elem() itemType := paramType.Elem()
// 如果当前参数是一个指针类型,则获取指针指向的结构体类型 // If the current parameter is a pointer type, get the struct type that the pointer points to
if itemType.Kind() == reflect.Ptr { if itemType.Kind() == reflect.Ptr {
itemType = itemType.Elem() // 获取指针指向的结构体类型 itemType = itemType.Elem() // Get the struct type that the pointer points to
} }
// Check if f implements Serialize interface // Check if f implements Serialize interface
// (检测传递的FaaS函数是否实现了Serialize接口)
if isSerialize(itemType) { if isSerialize(itemType) {
// 如果当前形参实现了Serialize接口则使用当前形参的序列化实现 // If the current parameter implements the Serialize interface, use the serialization implementation of the current parameter
serializeImpl = reflect.New(itemType).Interface().(Serialize) serializeImpl = reflect.New(itemType).Interface().(Serialize)
} else { } else {
// 如果当前形参没有实现Serialize接口则使用默认的序列化实现 // If the current parameter does not implement the Serialize interface, use the default serialization implementation
serializeImpl = defaultSerialize // Use global default implementation serializeImpl = defaultSerialize // Use global default implementation
} }
} else { } else {
// Other types are not supported // Other types are not supported
} }
// 将当前形参类型追加到argsType集合中 // Append the current parameter type to the argsType collection
argsType[i] = paramType argsType[i] = paramType
} }
if !containsKisFlow { if !containsKisFlow {
// 不包含kis.Flow类型的形参返回错误 // If the function parameter list does not contain a parameter of type kis.Flow, return an error
return nil, errors.New("function parameters must have kis.Flow param, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]") return nil, errors.New("function parameters must have kis.Flow param, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]")
} }
if !containsCtx { if !containsCtx {
// 不包含context.Context类型的形参返回错误 // If the function parameter list does not contain a parameter of type context.Context, return an error
return nil, errors.New("function parameters must have context, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]") return nil, errors.New("function parameters must have context, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]")
} }
// 返回FaaSDesc描述实例 // Return the FaaSDesc description instance
return &FaaSDesc{ return &FaaSDesc{
Serialize: serializeImpl, Serialize: serializeImpl,
FnName: fnName, FnName: fnName,
@ -129,26 +127,26 @@ func NewFaaSDesc(fnName string, f FaaS) (*FaaSDesc, error) {
}, nil }, nil
} }
// isFuncType 判断传递进来的 paramType 是否是函数类型 // isFuncType checks whether the provided paramType is a function type
func isFuncType(paramType reflect.Type) bool { func isFuncType(paramType reflect.Type) bool {
return paramType.Kind() == reflect.Func return paramType.Kind() == reflect.Func
} }
// isFlowType 判断传递进来的 paramType 是否是 kis.Flow 类型 // isFlowType checks whether the provided paramType is of type kis.Flow
func isFlowType(paramType reflect.Type) bool { func isFlowType(paramType reflect.Type) bool {
var flowInterfaceType = reflect.TypeOf((*Flow)(nil)).Elem() var flowInterfaceType = reflect.TypeOf((*Flow)(nil)).Elem()
return paramType.Implements(flowInterfaceType) return paramType.Implements(flowInterfaceType)
} }
// isContextType 判断传递进来的 paramType 是否是 context.Context 类型 // isContextType checks whether the provided paramType is of type context.Context
func isContextType(paramType reflect.Type) bool { func isContextType(paramType reflect.Type) bool {
typeName := paramType.Name() typeName := paramType.Name()
return strings.Contains(typeName, "Context") return strings.Contains(typeName, "Context")
} }
// isSliceType 判断传递进来的 paramType 是否是切片类型 // isSliceType checks whether the provided paramType is a slice type
func isSliceType(paramType reflect.Type) bool { func isSliceType(paramType reflect.Type) bool {
return paramType.Kind() == reflect.Slice return paramType.Kind() == reflect.Slice
} }

View File

@ -8,51 +8,51 @@ import (
) )
type Flow interface { type Flow interface {
// Run 调度Flow依次调度Flow中的Function并且执行 // Run schedules the Flow, sequentially dispatching and executing Functions in the Flow
Run(ctx context.Context) error Run(ctx context.Context) error
// Link 将Flow中的Function按照配置文件中的配置进行连接, 同时Flow的配置也会更新 // Link connects the Functions in the Flow according to the configuration in the config file, and the Flow's configuration will also be updated
Link(fConf *config.KisFuncConfig, fParams config.FParam) error Link(fConf *config.KisFuncConfig, fParams config.FParam) error
// AppendNewFunction 将一个新的Function追加到到Flow中 // AppendNewFunction appends a new Function to the Flow
AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error
// CommitRow 提交Flow数据到即将执行的Function层 // CommitRow submits Flow data to the upcoming Function layer
CommitRow(row interface{}) error CommitRow(row interface{}) error
// CommitRowBatch 提交Flow数据到即将执行的Function层(批量提交) // CommitRowBatch submits Flow data to the upcoming Function layer (batch submission)
// row: Must be a slice // row: Must be a slice
CommitRowBatch(row interface{}) error CommitRowBatch(row interface{}) error
// Input 得到flow当前执行Function的输入源数据 // Input gets the input source data of the currently executing Function in the Flow
Input() common.KisRowArr Input() common.KisRowArr
// GetName 得到Flow的名称 // GetName gets the name of the Flow
GetName() string GetName() string
// GetThisFunction 得到当前正在执行的Function // GetThisFunction gets the currently executing Function
GetThisFunction() Function GetThisFunction() Function
// GetThisFuncConf 得到当前正在执行的Function的配置 // GetThisFuncConf gets the configuration of the currently executing Function
GetThisFuncConf() *config.KisFuncConfig GetThisFuncConf() *config.KisFuncConfig
// GetConnector 得到当前正在执行的Function的Connector // GetConnector gets the Connector of the currently executing Function
GetConnector() (Connector, error) GetConnector() (Connector, error)
// GetConnConf 得到当前正在执行的Function的Connector的配置 // GetConnConf gets the configuration of the Connector of the currently executing Function
GetConnConf() (*config.KisConnConfig, error) GetConnConf() (*config.KisConnConfig, error)
// GetConfig 得到当前Flow的配置 // GetConfig gets the configuration of the current Flow
GetConfig() *config.KisFlowConfig GetConfig() *config.KisFlowConfig
// GetFuncConfigByName 得到当前Flow的配置 // GetFuncConfigByName gets the configuration of the current Flow by Function name
GetFuncConfigByName(funcName string) *config.KisFuncConfig GetFuncConfigByName(funcName string) *config.KisFuncConfig
// Next 当前Flow执行到的Function进入下一层Function所携带的Action动作 // Next carries the Action actions of the next layer Function that the current Flow is executing
Next(acts ...ActionFunc) error Next(acts ...ActionFunc) error
// GetCacheData 得到当前Flow的缓存数据 // GetCacheData gets the cached data of the current Flow
GetCacheData(key string) interface{} GetCacheData(key string) interface{}
// SetCacheData 设置当前Flow的缓存数据 // SetCacheData sets the cached data of the current Flow
SetCacheData(key string, value interface{}, Exp time.Duration) SetCacheData(key string, value interface{}, Exp time.Duration)
// GetMetaData 得到当前Flow的临时数据 // GetMetaData gets the temporary data of the current Flow
GetMetaData(key string) interface{} GetMetaData(key string) interface{}
// SetMetaData 设置当前Flow的临时数据 // SetMetaData sets the temporary data of the current Flow
SetMetaData(key string, value interface{}) SetMetaData(key string, value interface{})
// GetFuncParam 得到Flow的当前正在执行的Function的配置默认参数取出一对key-value // GetFuncParam gets the default parameters of the current Flow's currently executing Function, retrieving a key-value pair
GetFuncParam(key string) string GetFuncParam(key string) string
// GetFuncParamAll 得到Flow的当前正在执行的Function的配置默认参数取出全部Key-Value // GetFuncParamAll gets the default parameters of the current Flow's currently executing Function, retrieving all Key-Value pairs
GetFuncParamAll() config.FParam GetFuncParamAll() config.FParam
// GetFuncParamsAllFuncs 得到Flow中所有Function的FuncParams取出全部Key-Value // GetFuncParamsAllFuncs gets the FuncParams of all Functions in the Flow, retrieving all Key-Value pairs
GetFuncParamsAllFuncs() map[string]config.FParam GetFuncParamsAllFuncs() map[string]config.FParam
// Fork 得到Flow的一个副本(深拷贝) // Fork gets a copy of the Flow (deep copy)
Fork(ctx context.Context) Flow Fork(ctx context.Context) Flow
// GetId 得到Flow的Id // GetID gets the Id of the Flow
GetId() string GetID() string
} }

View File

@ -5,46 +5,46 @@ import (
"github.com/aceld/kis-flow/config" "github.com/aceld/kis-flow/config"
) )
// Function 流式计算基础计算模块KisFunction是一条流式计算的基本计算逻辑单元 // Function is the basic computation unit of streaming computation. KisFunction is a basic logical unit of streaming computation,
// 任意个KisFunction可以组合成一个KisFlow // any number of KisFunctions can be combined into a KisFlow
type Function interface { type Function interface {
// Call 执行流式计算逻辑 // Call executes the streaming computation logic
Call(ctx context.Context, flow Flow) error Call(ctx context.Context, flow Flow) error
// SetConfig 给当前Function实例配置策略 // SetConfig configures the current Function instance
SetConfig(s *config.KisFuncConfig) error SetConfig(s *config.KisFuncConfig) error
// GetConfig 获取当前Function实例配置策略 // GetConfig retrieves the configuration of the current Function instance
GetConfig() *config.KisFuncConfig GetConfig() *config.KisFuncConfig
// SetFlow 给当前Function实例设置所依赖的Flow实例 // SetFlow sets the Flow instance that the current Function instance depends on
SetFlow(f Flow) error SetFlow(f Flow) error
// GetFlow 获取当前Functioin实力所依赖的Flow // GetFlow retrieves the Flow instance that the current Function instance depends on
GetFlow() Flow GetFlow() Flow
// AddConnector 给当前Function实例添加一个Connector // AddConnector adds a Connector to the current Function instance
AddConnector(conn Connector) error AddConnector(conn Connector) error
// GetConnector 获取当前Function实例所关联的Connector // GetConnector retrieves the Connector associated with the current Function instance
GetConnector() Connector GetConnector() Connector
// CreateId 给当前Funciton实力生成一个随机的实例KisID // CreateId generates a random KisID for the current Function instance
CreateId() CreateId()
// GetId 获取当前Function的FID // GetID retrieves the FID of the current Function
GetId() string GetID() string
// GetPrevId 获取当前Function上一个Function节点FID // GetPrevId retrieves the FID of the previous Function node of the current Function
GetPrevId() string GetPrevId() string
// GetNextId 获取当前Function下一个Function节点FID // GetNextId retrieves the FID of the next Function node of the current Function
GetNextId() string GetNextId() string
// Next 返回下一层计算流Function如果当前层为最后一层则返回nil // Next returns the next layer of the computation flow Function. If the current layer is the last layer, it returns nil
Next() Function Next() Function
// Prev 返回上一层计算流Function如果当前层为最后一层则返回nil // Prev returns the previous layer of the computation flow Function. If the current layer is the last layer, it returns nil
Prev() Function Prev() Function
// SetN 设置下一层Function实例 // SetN sets the next Function instance
SetN(f Function) SetN(f Function)
// SetP 设置上一层Function实例 // SetP sets the previous Function instance
SetP(f Function) SetP(f Function)
// GetMetaData 得到当前Function的临时数据 // GetMetaData retrieves the temporary data of the current Function
GetMetaData(key string) interface{} GetMetaData(key string) interface{}
// SetMetaData 设置当前Function的临时数据 // SetMetaData sets the temporary data of the current Function
SetMetaData(key string, value interface{}) SetMetaData(key string, value interface{})
} }

View File

@ -12,37 +12,37 @@ import (
var _poolOnce sync.Once var _poolOnce sync.Once
// kisPool 用于管理全部的Function和Flow配置的池子 // KisPool manages all Function and Flow configurations
type kisPool struct { type KisPool struct {
fnRouter funcRouter // 全部的Function管理路由 fnRouter funcRouter // All Function management routes
fnLock sync.RWMutex // fnRouter fnLock sync.RWMutex // fnRouter lock
flowRouter flowRouter // 全部的flow对象 flowRouter flowRouter // All flow objects
flowLock sync.RWMutex // flowRouter flowLock sync.RWMutex // flowRouter lock
cInitRouter connInitRouter // 全部的Connector初始化路由 cInitRouter connInitRouter // All Connector initialization routes
ciLock sync.RWMutex // cInitRouter ciLock sync.RWMutex // cInitRouter lock
cTree connTree // 全部Connector管理路由 cTree connTree // All Connector management routes
cLock sync.RWMutex // cTree cLock sync.RWMutex // cTree lock
} }
// 单例 // Singleton
var _pool *kisPool var _pool *KisPool
// Pool 单例构造 // Pool Singleton constructor
func Pool() *kisPool { func Pool() *KisPool {
_poolOnce.Do(func() { _poolOnce.Do(func() {
//创建kisPool对象 // Create KisPool object
_pool = new(kisPool) _pool = &KisPool{}
// fnRouter初始化 // Initialize fnRouter
_pool.fnRouter = make(funcRouter) _pool.fnRouter = make(funcRouter)
// flowRouter初始化 // Initialize flowRouter
_pool.flowRouter = make(flowRouter) _pool.flowRouter = make(flowRouter)
// connTree初始化 // Initialize connTree
_pool.cTree = make(connTree) _pool.cTree = make(connTree)
_pool.cInitRouter = make(connInitRouter) _pool.cInitRouter = make(connInitRouter)
}) })
@ -50,8 +50,8 @@ func Pool() *kisPool {
return _pool return _pool
} }
func (pool *kisPool) AddFlow(name string, flow Flow) { func (pool *KisPool) AddFlow(name string, flow Flow) {
pool.flowLock.Lock() // 写锁 pool.flowLock.Lock() // Write lock
defer pool.flowLock.Unlock() defer pool.flowLock.Unlock()
if _, ok := pool.flowRouter[name]; !ok { if _, ok := pool.flowRouter[name]; !ok {
@ -64,8 +64,8 @@ func (pool *kisPool) AddFlow(name string, flow Flow) {
log.Logger().InfoF("Add FlowRouter FlowName=%s", name) log.Logger().InfoF("Add FlowRouter FlowName=%s", name)
} }
func (pool *kisPool) GetFlow(name string) Flow { func (pool *KisPool) GetFlow(name string) Flow {
pool.flowLock.RLock() // 读锁 pool.flowLock.RLock() // Read lock
defer pool.flowLock.RUnlock() defer pool.flowLock.RUnlock()
if flow, ok := pool.flowRouter[name]; ok { if flow, ok := pool.flowRouter[name]; ok {
@ -75,20 +75,20 @@ func (pool *kisPool) GetFlow(name string) Flow {
} }
} }
// FaaS 注册 Function 计算业务逻辑, 通过Function Name 索引及注册 // FaaS registers Function computation business logic, indexed and registered by Function Name
func (pool *kisPool) FaaS(fnName string, f FaaS) { func (pool *KisPool) FaaS(fnName string, f FaaS) {
// 当注册FaaS计算逻辑回调时创建一个FaaSDesc描述对象 // When registering the FaaS computation logic callback, create a FaaSDesc description object
faaSDesc, err := NewFaaSDesc(fnName, f) faaSDesc, err := NewFaaSDesc(fnName, f)
if err != nil { if err != nil {
panic(err) panic(err)
} }
pool.fnLock.Lock() // 写锁 pool.fnLock.Lock() // Write lock
defer pool.fnLock.Unlock() defer pool.fnLock.Unlock()
if _, ok := pool.fnRouter[fnName]; !ok { if _, ok := pool.fnRouter[fnName]; !ok {
// 将FaaSDesc描述对象注册到fnRouter中 // Register the FaaSDesc description object to fnRouter
pool.fnRouter[fnName] = faaSDesc pool.fnRouter[fnName] = faaSDesc
} else { } else {
errString := fmt.Sprintf("KisPoll FaaS Repeat FuncName=%s", fnName) errString := fmt.Sprintf("KisPoll FaaS Repeat FuncName=%s", fnName)
@ -98,33 +98,33 @@ func (pool *kisPool) FaaS(fnName string, f FaaS) {
log.Logger().InfoF("Add KisPool FuncName=%s", fnName) log.Logger().InfoF("Add KisPool FuncName=%s", fnName)
} }
// CallFunction 调度 Function // CallFunction schedules Function
func (pool *kisPool) CallFunction(ctx context.Context, fnName string, flow Flow) error { func (pool *KisPool) CallFunction(ctx context.Context, fnName string, flow Flow) error {
pool.fnLock.RLock() // 读锁 pool.fnLock.RLock() // Read lock
defer pool.fnLock.RUnlock() defer pool.fnLock.RUnlock()
if funcDesc, ok := pool.fnRouter[fnName]; ok { if funcDesc, ok := pool.fnRouter[fnName]; ok {
// 被调度Function的形参列表 // Parameters list for the scheduled Function
params := make([]reflect.Value, 0, funcDesc.ArgNum) params := make([]reflect.Value, 0, funcDesc.ArgNum)
for _, argType := range funcDesc.ArgsType { for _, argType := range funcDesc.ArgsType {
// 如果是Flow类型形参则将 flow的值传入 // If it is a Flow type parameter, pass in the value of flow
if isFlowType(argType) { if isFlowType(argType) {
params = append(params, reflect.ValueOf(flow)) params = append(params, reflect.ValueOf(flow))
continue continue
} }
// 如果是Context类型形参则将 ctx的值传入 // If it is a Context type parameter, pass in the value of ctx
if isContextType(argType) { if isContextType(argType) {
params = append(params, reflect.ValueOf(ctx)) params = append(params, reflect.ValueOf(ctx))
continue continue
} }
// 如果是Slice类型形参则将 flow.Input()的值传入 // If it is a Slice type parameter, pass in the value of flow.Input()
if isSliceType(argType) { if isSliceType(argType) {
// 将flow.Input()中的原始数据反序列化为argType类型的数据 // Deserialize the raw data in flow.Input() to data of type argType
value, err := funcDesc.Serialize.UnMarshal(flow.Input(), argType) value, err := funcDesc.Serialize.UnMarshal(flow.Input(), argType)
if err != nil { if err != nil {
log.Logger().ErrorFX(ctx, "funcDesc.Serialize.DecodeParam err=%v", err) log.Logger().ErrorFX(ctx, "funcDesc.Serialize.DecodeParam err=%v", err)
@ -135,20 +135,20 @@ func (pool *kisPool) CallFunction(ctx context.Context, fnName string, flow Flow)
} }
// 传递的参数既不是Flow类型也不是Context类型也不是Slice类型则默认给到零值 // If the passed parameter is neither a Flow type, nor a Context type, nor a Slice type, it defaults to zero value
params = append(params, reflect.Zero(argType)) params = append(params, reflect.Zero(argType))
} }
// 调用当前Function 的计算逻辑 // Call the computation logic of the current Function
retValues := funcDesc.FuncValue.Call(params) retValues := funcDesc.FuncValue.Call(params)
// 取出第一个返回值如果是nil则返回nil // Extract the first return value, if it is nil, return nil
ret := retValues[0].Interface() ret := retValues[0].Interface()
if ret == nil { if ret == nil {
return nil return nil
} }
// 如果返回值是error类型则返回error // If the return value is of type error, return error
return retValues[0].Interface().(error) return retValues[0].Interface().(error)
} }
@ -158,9 +158,9 @@ func (pool *kisPool) CallFunction(ctx context.Context, fnName string, flow Flow)
return errors.New("FuncName: " + fnName + " Can not find in NsPool, Not Added.") return errors.New("FuncName: " + fnName + " Can not find in NsPool, Not Added.")
} }
// CaaSInit 注册Connector初始化业务 // CaaSInit registers Connector initialization business
func (pool *kisPool) CaaSInit(cname string, c ConnInit) { func (pool *KisPool) CaaSInit(cname string, c ConnInit) {
pool.ciLock.Lock() // 写锁 pool.ciLock.Lock() // Write lock
defer pool.ciLock.Unlock() defer pool.ciLock.Unlock()
if _, ok := pool.cInitRouter[cname]; !ok { if _, ok := pool.cInitRouter[cname]; !ok {
@ -173,9 +173,9 @@ func (pool *kisPool) CaaSInit(cname string, c ConnInit) {
log.Logger().InfoF("Add KisPool CaaSInit CName=%s", cname) log.Logger().InfoF("Add KisPool CaaSInit CName=%s", cname)
} }
// CallConnInit 调度 ConnInit // CallConnInit schedules ConnInit
func (pool *kisPool) CallConnInit(conn Connector) error { func (pool *KisPool) CallConnInit(conn Connector) error {
pool.ciLock.RLock() // 读锁 pool.ciLock.RLock() // Read lock
defer pool.ciLock.RUnlock() defer pool.ciLock.RUnlock()
init, ok := pool.cInitRouter[conn.GetName()] init, ok := pool.cInitRouter[conn.GetName()]
@ -187,16 +187,16 @@ func (pool *kisPool) CallConnInit(conn Connector) error {
return init(conn) return init(conn)
} }
// CaaS 注册Connector Call业务 // CaaS registers Connector Call business
func (pool *kisPool) CaaS(cname string, fname string, mode common.KisMode, c CaaS) { func (pool *KisPool) CaaS(cname string, fname string, mode common.KisMode, c CaaS) {
pool.cLock.Lock() // 写锁 pool.cLock.Lock() // Write lock
defer pool.cLock.Unlock() defer pool.cLock.Unlock()
if _, ok := pool.cTree[cname]; !ok { if _, ok := pool.cTree[cname]; !ok {
//cid 首次注册,不存在,创建二级树NsConnSL //cid First registration, does not exist, create a second-level tree NsConnSL
pool.cTree[cname] = make(connSL) pool.cTree[cname] = make(connSL)
//初始化各类型FunctionMode // Initialize various FunctionMode
pool.cTree[cname][common.S] = make(connFuncRouter) pool.cTree[cname][common.S] = make(connFuncRouter)
pool.cTree[cname][common.L] = make(connFuncRouter) pool.cTree[cname][common.L] = make(connFuncRouter)
} }
@ -211,9 +211,9 @@ func (pool *kisPool) CaaS(cname string, fname string, mode common.KisMode, c Caa
log.Logger().InfoF("Add KisPool CaaS CName=%s, FName=%s, Mode =%s", cname, fname, mode) log.Logger().InfoF("Add KisPool CaaS CName=%s, FName=%s, Mode =%s", cname, fname, mode)
} }
// CallConnector 调度 Connector // CallConnector schedules Connector
func (pool *kisPool) CallConnector(ctx context.Context, flow Flow, conn Connector, args interface{}) (interface{}, error) { func (pool *KisPool) CallConnector(ctx context.Context, flow Flow, conn Connector, args interface{}) (interface{}, error) {
pool.cLock.RLock() // 读锁 pool.cLock.RLock() // Read lock
defer pool.cLock.RUnlock() defer pool.cLock.RUnlock()
fn := flow.GetThisFunction() fn := flow.GetThisFunction()
fnConf := fn.GetConfig() fnConf := fn.GetConfig()
@ -228,9 +228,9 @@ func (pool *kisPool) CallConnector(ctx context.Context, flow Flow, conn Connecto
return nil, errors.New(fmt.Sprintf("CName:%s FName:%s mode:%s Can not find in KisPool, Not Added.", conn.GetName(), fnConf.FName, mode)) return nil, errors.New(fmt.Sprintf("CName:%s FName:%s mode:%s Can not find in KisPool, Not Added.", conn.GetName(), fnConf.FName, mode))
} }
// GetFlows 得到全部的Flow // GetFlows retrieves all Flows
func (pool *kisPool) GetFlows() []Flow { func (pool *KisPool) GetFlows() []Flow {
pool.flowLock.RLock() // 读锁 pool.flowLock.RLock() // Read lock
defer pool.flowLock.RUnlock() defer pool.flowLock.RUnlock()
var flows []Flow var flows []Flow

View File

@ -5,14 +5,12 @@ import (
"github.com/aceld/kis-flow/common" "github.com/aceld/kis-flow/common"
) )
// FaaS 定义移植到 faas.go 中
/* /*
Function Call Function Call
*/ */
// funcRouter // funcRouter
// key: Function Name // key: Function Name
// value: FaaSDesc 回调自定义业务的描述 // value: FaaSDesc callback description for custom business
type funcRouter map[string]*FaaSDesc type funcRouter map[string]*FaaSDesc
// flowRouter // flowRouter
@ -23,30 +21,31 @@ type flowRouter map[string]Flow
/* /*
Connector Init Connector Init
*/ */
// ConnInit Connector 第三方挂载存储初始化 // ConnInit Connector third-party storage initialization
type ConnInit func(conn Connector) error type ConnInit func(conn Connector) error
// connInitRouter // connInitRouter
// key: // key: Connector Name
// value: ConnInit
type connInitRouter map[string]ConnInit type connInitRouter map[string]ConnInit
/* /*
Connector Call Connector Call
*/ */
// CaaS Connector的存储读取业务实现 // CaaS Connector storage read/write business implementation
type CaaS func(context.Context, Connector, Function, Flow, interface{}) (interface{}, error) type CaaS func(context.Context, Connector, Function, Flow, interface{}) (interface{}, error)
// connFuncRouter 通过FunctionName索引到CaaS回调存储业务的映射关系 // connFuncRouter Maps CaaS callback storage business to FunctionName
// key: Function Name // key: Function Name
// value: Connector的存储读取业务实现 // value: Connector storage read/write business implementation
type connFuncRouter map[string]CaaS type connFuncRouter map[string]CaaS
// connSL 通过KisMode 将connFuncRouter分为两个子树 // connSL Splits connFuncRouter into two subtrees based on KisMode
// key: Function KisMode S/L // key: Function KisMode S/L
// value: NsConnRouter // value: connFuncRouter
type connSL map[common.KisMode]connFuncRouter type connSL map[common.KisMode]connFuncRouter
// connTree // connTree
// key: Connector Name // key: Connector Name
// value: connSL 二级树 // value: connSL second-level tree
type connTree map[string]connSL type connTree map[string]connSL

View File

@ -6,18 +6,18 @@ import (
"reflect" "reflect"
) )
// Serialize 数据序列化接口 // Serialize Data serialization interface
type Serialize interface { type Serialize interface {
// UnMarshal 用于将 KisRowArr 反序列化为指定类型的值。 // UnMarshal is used to deserialize KisRowArr to a value of the specified type.
UnMarshal(common.KisRowArr, reflect.Type) (reflect.Value, error) UnMarshal(common.KisRowArr, reflect.Type) (reflect.Value, error)
// Marshal 用于将指定类型的值序列化为 KisRowArr。 // Marshal is used to serialize a value of the specified type to KisRowArr.
Marshal(interface{}) (common.KisRowArr, error) Marshal(interface{}) (common.KisRowArr, error)
} }
// defaultSerialize KisFlow提供的默认序列化实现(开发者可以自定义) // defaultSerialize Default serialization implementation provided by KisFlow (developers can customize)
var defaultSerialize = &serialize.DefaultSerialize{} var defaultSerialize = &serialize.DefaultSerialize{}
// isSerialize 判断传递进来的 paramType 是否实现了 Serialize 接口 // isSerialize checks if the provided paramType implements the Serialize interface
func isSerialize(paramType reflect.Type) bool { func isSerialize(paramType reflect.Type) bool {
return paramType.Implements(reflect.TypeOf((*Serialize)(nil)).Elem()) return paramType.Implements(reflect.TypeOf((*Serialize)(nil)).Elem())
} }

View File

@ -6,7 +6,7 @@ import (
"sync" "sync"
) )
// kisDefaultLog 默认提供的日志对象 // kisDefaultLog Default provided log object
type kisDefaultLog struct { type kisDefaultLog struct {
debugMode bool debugMode bool
mu sync.Mutex mu sync.Mutex
@ -60,7 +60,7 @@ func (log *kisDefaultLog) DebugFX(ctx context.Context, str string, v ...interfac
} }
func init() { func init() {
// 如果没有设置Logger, 则启动时使用默认的kisDefaultLog对象 // If no logger is set, use the default kisDefaultLog object at startup
if Logger() == nil { if Logger() == nil {
SetLogger(&kisDefaultLog{}) SetLogger(&kisDefaultLog{})
} }

View File

@ -3,33 +3,33 @@ package log
import "context" import "context"
type KisLogger interface { type KisLogger interface {
// InfoFX 有上下文的Info级别日志接口, format字符串格式 // InfoFX with context Info-level log interface, format string format
InfoFX(ctx context.Context, str string, v ...interface{}) InfoFX(ctx context.Context, str string, v ...interface{})
// ErrorFX 有上下文的Error级别日志接口, format字符串格式 // ErrorFX with context Error-level log interface, format string format
ErrorFX(ctx context.Context, str string, v ...interface{}) ErrorFX(ctx context.Context, str string, v ...interface{})
// DebugFX 有上下文的Debug级别日志接口, format字符串格式 // DebugFX with context Debug-level log interface, format string format
DebugFX(ctx context.Context, str string, v ...interface{}) DebugFX(ctx context.Context, str string, v ...interface{})
// InfoF 无上下文的Info级别日志接口, format字符串格式 // InfoF without context Info-level log interface, format string format
InfoF(str string, v ...interface{}) InfoF(str string, v ...interface{})
// ErrorF 无上下文的Error级别日志接口, format字符串格式 // ErrorF without context Error-level log interface, format string format
ErrorF(str string, v ...interface{}) ErrorF(str string, v ...interface{})
// DebugF 无上下文的Debug级别日志接口, format字符串格式 // DebugF without context Debug-level log interface, format string format
DebugF(str string, v ...interface{}) DebugF(str string, v ...interface{})
// SetDebugMode 设置Debug模式 // SetDebugMode set Debug mode
SetDebugMode(enable bool) SetDebugMode(enable bool)
} }
// kisLog 默认的KisLog 对象, 提供默认的日志打印方式, 均是打印在标准输出上。 // kisLog Default KisLog object, providing default log printing methods, all of which print to standard output.
var kisLog KisLogger var kisLog KisLogger
// SetLogger 设置KisLog对象, 可以是用户自定义的Logger对象 // SetLogger set KisLog object, can be a user-defined Logger object
func SetLogger(newlog KisLogger) { func SetLogger(newlog KisLogger) {
kisLog = newlog kisLog = newlog
} }
// Logger 获取到kisLog对象 // Logger get the kisLog object
func Logger() KisLogger { func Logger() KisLogger {
return kisLog return kisLog
} }

View File

@ -9,32 +9,32 @@ import (
"net/http" "net/http"
) )
// kisMetrics kisFlow的Prometheus监控指标 // KisMetrics kisFlow's Prometheus monitoring metrics
type kisMetrics struct { type KisMetrics struct {
// 数据数量总量 // Total data quantity
DataTotal prometheus.Counter DataTotal prometheus.Counter
// 各Flow处理数据总量 // Total data processed by each Flow
FlowDataTotal *prometheus.GaugeVec FlowDataTotal *prometheus.GaugeVec
// Flow被调度次数 // Flow scheduling counts
FlowScheduleCntsToTal *prometheus.GaugeVec FlowScheduleCntsToTal *prometheus.GaugeVec
// Function被调度次数 // Function scheduling counts
FuncScheduleCntsTotal *prometheus.GaugeVec FuncScheduleCntsTotal *prometheus.GaugeVec
// Function执行时间 // Function execution time
FunctionDuration *prometheus.HistogramVec FunctionDuration *prometheus.HistogramVec
// Flow执行时间 // Flow execution time
FlowDuration *prometheus.HistogramVec FlowDuration *prometheus.HistogramVec
} }
var Metrics *kisMetrics var Metrics *KisMetrics
// RunMetricsService 启动Prometheus监控服务 // RunMetricsService starts the Prometheus monitoring service
func RunMetricsService(serverAddr string) error { func RunMetricsService(serverAddr string) error {
// 注册Prometheus 监控路由路径 // Register Prometheus monitoring route path
http.Handle(common.METRICS_ROUTE, promhttp.Handler()) http.Handle(common.MetricsRoute, promhttp.Handler())
// 启动HttpServer // Start HttpServer
err := http.ListenAndServe(serverAddr, nil) //多个进程不可监听同一个端口 err := http.ListenAndServe(serverAddr, nil) // Multiple processes cannot listen on the same port
if err != nil { if err != nil {
log.Logger().ErrorF("RunMetricsService err = %s\n", err) log.Logger().ErrorF("RunMetricsService err = %s\n", err)
} }
@ -43,64 +43,64 @@ func RunMetricsService(serverAddr string) error {
} }
func InitMetrics() { func InitMetrics() {
Metrics = new(kisMetrics) Metrics = new(KisMetrics)
// DataTotal初始化Counter // Initialize DataTotal Counter
Metrics.DataTotal = prometheus.NewCounter(prometheus.CounterOpts{ Metrics.DataTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: common.COUNTER_KISFLOW_DATA_TOTAL_NAME, Name: common.CounterKisflowDataTotalName,
Help: common.COUNTER_KISFLOW_DATA_TOTAL_HELP, Help: common.CounterKisflowDataTotalHelp,
}) })
// FlowDataTotal初始化GaugeVec // Initialize FlowDataTotal GaugeVec
Metrics.FlowDataTotal = prometheus.NewGaugeVec( Metrics.FlowDataTotal = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: common.GANGE_FLOW_DATA_TOTAL_NAME, Name: common.GamgeFlowDataTotalName,
Help: common.GANGE_FLOW_DATA_TOTAL_HELP, Help: common.GamgeFlowDataTotalHelp,
}, },
// 标签名称 // Label names
[]string{common.LABEL_FLOW_NAME}, []string{common.LabelFlowName},
) )
// FlowScheduleCntsToTal初始化GaugeVec // Initialize FlowScheduleCntsToTal GaugeVec
Metrics.FlowScheduleCntsToTal = prometheus.NewGaugeVec( Metrics.FlowScheduleCntsToTal = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: common.GANGE_FLOW_SCHE_CNTS_NAME, Name: common.GangeFlowScheCntsName,
Help: common.GANGE_FLOW_SCHE_CNTS_HELP, Help: common.GangeFlowScheCntsHelp,
}, },
//标签名称 // Label names
[]string{common.LABEL_FLOW_NAME}, []string{common.LabelFlowName},
) )
// FuncScheduleCntsTotal初始化GaugeVec // Initialize FuncScheduleCntsTotal GaugeVec
Metrics.FuncScheduleCntsTotal = prometheus.NewGaugeVec( Metrics.FuncScheduleCntsTotal = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: common.GANGE_FUNC_SCHE_CNTS_NAME, Name: common.GangeFuncScheCntsName,
Help: common.GANGE_FUNC_SCHE_CNTS_HELP, Help: common.GangeFuncScheCntsHelp,
}, },
//标签名称 // Label names
[]string{common.LABEL_FUNCTION_NAME, common.LABEL_FUNCTION_MODE}, []string{common.LabelFunctionName, common.LabelFunctionMode},
) )
// FunctionDuration初始化HistogramVec // Initialize FunctionDuration HistogramVec
Metrics.FunctionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Metrics.FunctionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: common.HISTOGRAM_FUNCTION_DURATION_NAME, Name: common.HistogramFunctionDurationName,
Help: common.HISTOGRAM_FUNCTION_DURATION_HELP, Help: common.HistogramFunctionDurationHelp,
Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000}, //单位ms,最大半分钟 Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000}, // Unit: ms, maximum half a minute
}, },
[]string{common.LABEL_FUNCTION_NAME, common.LABEL_FUNCTION_MODE}, []string{common.LabelFunctionName, common.LabelFunctionMode},
) )
// FlowDuration初始化HistogramVec // Initialize FlowDuration HistogramVec
Metrics.FlowDuration = prometheus.NewHistogramVec( Metrics.FlowDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: common.HISTOGRAM_FLOW_DURATION_NAME, Name: common.HistogramFlowDurationName,
Help: common.HISTOGRAM_FLOW_DURATION_HELP, Help: common.HistogramFlowDurationHelp,
Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000, 60000}, //单位ms,最大1分钟 Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000, 60000}, // Unit: ms, maximum 1 minute
}, },
[]string{common.LABEL_FLOW_NAME}, []string{common.LabelFlowName},
) )
// 注册Metrics // Register Metrics
prometheus.MustRegister(Metrics.DataTotal) prometheus.MustRegister(Metrics.DataTotal)
prometheus.MustRegister(Metrics.FlowDataTotal) prometheus.MustRegister(Metrics.FlowDataTotal)
prometheus.MustRegister(Metrics.FlowScheduleCntsToTal) prometheus.MustRegister(Metrics.FlowScheduleCntsToTal)
@ -109,13 +109,13 @@ func InitMetrics() {
prometheus.MustRegister(Metrics.FlowDuration) prometheus.MustRegister(Metrics.FlowDuration)
} }
// RunMetrics 启动Prometheus指标服务 // RunMetrics starts the Prometheus metrics service
func RunMetrics() { func RunMetrics() {
// 初始化Prometheus指标 // Initialize Prometheus metrics
InitMetrics() InitMetrics()
if config.GlobalConfig.EnableProm == true && config.GlobalConfig.PrometheusListen == true { if config.GlobalConfig.EnableProm == true && config.GlobalConfig.PrometheusListen == true {
// 启动Prometheus指标Metrics服务 // Start Prometheus metrics service
go RunMetricsService(config.GlobalConfig.PrometheusServe) go RunMetricsService(config.GlobalConfig.PrometheusServe)
} }
} }

View File

@ -1,9 +1,14 @@
/*
DefaultSerialize 实现了 Serialize 接口用于将 KisRowArr 序列化为指定类型的值或将指定类型的值序列化为 KisRowArr
这部分是KisFlow默认提供的序列化办法默认均是josn序列化开发者可以根据自己的需求实现自己的序列化办法
*/
package serialize package serialize
/*
DefaultSerialize implements the Serialize interface,
which is used to serialize KisRowArr into a specified type, or serialize a specified type into KisRowArr.
This section is the default serialization method provided by KisFlow, and it defaults to json serialization.
Developers can implement their own serialization methods according to their needs.
*/
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
@ -13,35 +18,35 @@ import (
type DefaultSerialize struct{} type DefaultSerialize struct{}
// UnMarshal 用于将 KisRowArr 反序列化为指定类型的值。 // UnMarshal is used to deserialize KisRowArr into a specified type.
func (f *DefaultSerialize) UnMarshal(arr common.KisRowArr, r reflect.Type) (reflect.Value, error) { func (f *DefaultSerialize) UnMarshal(arr common.KisRowArr, r reflect.Type) (reflect.Value, error) {
// 确保传入的类型是一个切片 // Ensure the passed-in type is a slice
if r.Kind() != reflect.Slice { if r.Kind() != reflect.Slice {
return reflect.Value{}, fmt.Errorf("r must be a slice") return reflect.Value{}, fmt.Errorf("r must be a slice")
} }
slice := reflect.MakeSlice(r, 0, len(arr)) slice := reflect.MakeSlice(r, 0, len(arr))
// 遍历每个元素并尝试反序列化 // Iterate through each element and attempt deserialization
for _, row := range arr { for _, row := range arr {
var elem reflect.Value var elem reflect.Value
var err error var err error
// 尝试断言为结构体或指针 // Try to assert as a struct or pointer
elem, err = unMarshalStruct(row, r.Elem()) elem, err = unMarshalStruct(row, r.Elem())
if err == nil { if err == nil {
slice = reflect.Append(slice, elem) slice = reflect.Append(slice, elem)
continue continue
} }
// 尝试直接反序列化字符串 // Try to directly deserialize the string
elem, err = unMarshalJsonString(row, r.Elem()) elem, err = unMarshalJsonString(row, r.Elem())
if err == nil { if err == nil {
slice = reflect.Append(slice, elem) slice = reflect.Append(slice, elem)
continue continue
} }
// 尝试先序列化为 JSON 再反序列化 // Try to serialize to JSON first and then deserialize
elem, err = unMarshalJsonStruct(row, r.Elem()) elem, err = unMarshalJsonStruct(row, r.Elem())
if err == nil { if err == nil {
slice = reflect.Append(slice, elem) slice = reflect.Append(slice, elem)
@ -53,9 +58,9 @@ func (f *DefaultSerialize) UnMarshal(arr common.KisRowArr, r reflect.Type) (refl
return slice, nil return slice, nil
} }
// 尝试断言为结构体或指针 // Try to assert as a struct or pointer
func unMarshalStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { func unMarshalStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) {
// 检查 row 是否为结构体或结构体指针类型 // Check if row is of struct or struct pointer type
rowType := reflect.TypeOf(row) rowType := reflect.TypeOf(row)
if rowType == nil { if rowType == nil {
return reflect.Value{}, fmt.Errorf("row is nil pointer") return reflect.Value{}, fmt.Errorf("row is nil pointer")
@ -64,41 +69,41 @@ func unMarshalStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, e
return reflect.Value{}, fmt.Errorf("row must be a struct or struct pointer type") return reflect.Value{}, fmt.Errorf("row must be a struct or struct pointer type")
} }
// 如果 row 是指针类型,则获取它指向的类型 // If row is a pointer type, get the type it points to
if rowType.Kind() == reflect.Ptr { if rowType.Kind() == reflect.Ptr {
// 空指针 // Nil pointer
if reflect.ValueOf(row).IsNil() { if reflect.ValueOf(row).IsNil() {
return reflect.Value{}, fmt.Errorf("row is nil pointer") return reflect.Value{}, fmt.Errorf("row is nil pointer")
} }
// 解引用 // Dereference
row = reflect.ValueOf(row).Elem().Interface() row = reflect.ValueOf(row).Elem().Interface()
// 拿到解引用后的类型 // Get the type after dereferencing
rowType = reflect.TypeOf(row) rowType = reflect.TypeOf(row)
} }
// 检查是否可以将 row 断言为 elemType(目标类型) // Check if row can be asserted to elemType(target type)
if !rowType.AssignableTo(elemType) { if !rowType.AssignableTo(elemType) {
return reflect.Value{}, fmt.Errorf("row type cannot be asserted to elemType") return reflect.Value{}, fmt.Errorf("row type cannot be asserted to elemType")
} }
// 将 row 转换为 reflect.Value 并返回 // Convert row to reflect.Value and return
return reflect.ValueOf(row), nil return reflect.ValueOf(row), nil
} }
// 尝试直接反序列化字符串(将Json字符串 反序列化为 结构体) // Try to directly deserialize the string(Deserialize the Json string into a struct)
func unMarshalJsonString(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { func unMarshalJsonString(row common.KisRow, elemType reflect.Type) (reflect.Value, error) {
// 判断源数据是否可以断言成string // Check if the source data can be asserted as a string
str, ok := row.(string) str, ok := row.(string)
if !ok { if !ok {
return reflect.Value{}, fmt.Errorf("not a string") return reflect.Value{}, fmt.Errorf("not a string")
} }
// 创建一个新的结构体实例,用于存储反序列化后的值 // Create a new struct instance to store the deserialized value
elem := reflect.New(elemType).Elem() elem := reflect.New(elemType).Elem()
// 尝试将json字符串反序列化为结构体。 // Try to deserialize the json string into a struct.
if err := json.Unmarshal([]byte(str), elem.Addr().Interface()); err != nil { if err := json.Unmarshal([]byte(str), elem.Addr().Interface()); err != nil {
return reflect.Value{}, fmt.Errorf("failed to unmarshal string to struct: %v", err) return reflect.Value{}, fmt.Errorf("failed to unmarshal string to struct: %v", err)
} }
@ -106,18 +111,18 @@ func unMarshalJsonString(row common.KisRow, elemType reflect.Type) (reflect.Valu
return elem, nil return elem, nil
} }
// 尝试先序列化为 JSON 再反序列化(将结构体转换成Json字符串再将Json字符串 反序列化为 结构体) // Try to serialize to JSON first and then deserialize(Serialize the struct to Json string, and then deserialize the Json string into a struct)
func unMarshalJsonStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { func unMarshalJsonStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) {
// 将 row 序列化为 JSON 字符串 // Serialize row to JSON string
jsonBytes, err := json.Marshal(row) jsonBytes, err := json.Marshal(row)
if err != nil { if err != nil {
return reflect.Value{}, fmt.Errorf("failed to marshal row to JSON: %v ", err) return reflect.Value{}, fmt.Errorf("failed to marshal row to JSON: %v ", err)
} }
// 创建一个新的结构体实例,用于存储反序列化后的值 // Create a new struct instance to store the deserialized value
elem := reflect.New(elemType).Interface() elem := reflect.New(elemType).Interface()
// 将 JSON 字符串反序列化为结构体 // Deserialize the JSON string into a struct
if err := json.Unmarshal(jsonBytes, elem); err != nil { if err := json.Unmarshal(jsonBytes, elem); err != nil {
return reflect.Value{}, fmt.Errorf("failed to unmarshal JSON to element: %v ", err) return reflect.Value{}, fmt.Errorf("failed to unmarshal JSON to element: %v ", err)
} }
@ -125,7 +130,7 @@ func unMarshalJsonStruct(row common.KisRow, elemType reflect.Type) (reflect.Valu
return reflect.ValueOf(elem).Elem(), nil return reflect.ValueOf(elem).Elem(), nil
} }
// Marshal 用于将指定类型的值序列化为 KisRowArr(json 序列化)。 // Marshal is used to serialize a specified type into KisRowArr(json serialization).
func (f *DefaultSerialize) Marshal(i interface{}) (common.KisRowArr, error) { func (f *DefaultSerialize) Marshal(i interface{}) (common.KisRowArr, error) {
var arr common.KisRowArr var arr common.KisRowArr
@ -133,7 +138,7 @@ func (f *DefaultSerialize) Marshal(i interface{}) (common.KisRowArr, error) {
case reflect.Slice, reflect.Array: case reflect.Slice, reflect.Array:
slice := reflect.ValueOf(i) slice := reflect.ValueOf(i)
for i := 0; i < slice.Len(); i++ { for i := 0; i < slice.Len(); i++ {
// 序列化每个元素为 JSON 字符串,并将其添加到切片中。 // Serialize each element to a JSON string and append it to the slice.
jsonBytes, err := json.Marshal(slice.Index(i).Interface()) jsonBytes, err := json.Marshal(slice.Index(i).Interface())
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err) return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err)
@ -141,7 +146,7 @@ func (f *DefaultSerialize) Marshal(i interface{}) (common.KisRowArr, error) {
arr = append(arr, string(jsonBytes)) arr = append(arr, string(jsonBytes))
} }
default: default:
// 如果不是切片或数组类型,则直接序列化整个结构体为 JSON 字符串。 // If it's not a slice or array type, serialize the entire struct to a JSON string directly.
jsonBytes, err := json.Marshal(i) jsonBytes, err := json.Marshal(i)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err) return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err)

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName1 fname: funcName1
fmode: Verify fmode: Verify
source: source:
name: 公众号抖音商城户订单数据 name: TikTok-Order
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName2 fname: funcName2
fmode: Save fmode: Save
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName3 fname: funcName3
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -12,7 +12,7 @@ func AbortFuncHandler(ctx context.Context, flow kis.Flow) error {
fmt.Println("---> Call AbortFuncHandler ----") fmt.Println("---> Call AbortFuncHandler ----")
for _, row := range flow.Input() { for _, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
} }

View File

@ -12,13 +12,11 @@ func DataReuseFuncHandler(ctx context.Context, flow kis.Flow) error {
fmt.Println("---> Call DataReuseFuncHandler ----") fmt.Println("---> Call DataReuseFuncHandler ----")
for index, row := range flow.Input() { for index, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
// 计算结果数据
resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index)
// 提交结果数据
_ = flow.CommitRow(resultStr) _ = flow.CommitRow(resultStr)
} }

View File

@ -13,14 +13,11 @@ func FuncDemo1Handler(ctx context.Context, flow kis.Flow) error {
fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) fmt.Printf("Params = %+v\n", flow.GetFuncParamAll())
for index, row := range flow.Input() { for index, row := range flow.Input() {
// 打印数据 str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row)
fmt.Println(str) fmt.Println(str)
// 计算结果数据
resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index)
// 提交结果数据
_ = flow.CommitRow(resultStr) _ = flow.CommitRow(resultStr)
} }

View File

@ -14,7 +14,7 @@ func FuncDemo2Handler(ctx context.Context, flow kis.Flow) error {
fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) fmt.Printf("Params = %+v\n", flow.GetFuncParamAll())
for index, row := range flow.Input() { for index, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
conn, err := flow.GetConnector() conn, err := flow.GetConnector()
@ -28,10 +28,8 @@ func FuncDemo2Handler(ctx context.Context, flow kis.Flow) error {
return err return err
} }
// 计算结果数据
resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index)
// 提交结果数据
_ = flow.CommitRow(resultStr) _ = flow.CommitRow(resultStr)
} }

View File

@ -13,7 +13,7 @@ func FuncDemo3Handler(ctx context.Context, flow kis.Flow) error {
fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) fmt.Printf("Params = %+v\n", flow.GetFuncParamAll())
for _, row := range flow.Input() { for _, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
} }

View File

@ -12,13 +12,11 @@ func FuncDemo4Handler(ctx context.Context, flow kis.Flow) error {
fmt.Println("---> Call FuncDemo4Handler ----") fmt.Println("---> Call FuncDemo4Handler ----")
for index, row := range flow.Input() { for index, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
// 计算结果数据
resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index)
// 提交结果数据
_ = flow.CommitRow(resultStr) _ = flow.CommitRow(resultStr)
} }

View File

@ -12,7 +12,7 @@ func JumpFuncHandler(ctx context.Context, flow kis.Flow) error {
fmt.Println("---> Call JumpFuncHandler ----") fmt.Println("---> Call JumpFuncHandler ----")
for _, row := range flow.Input() { for _, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
} }

View File

@ -12,7 +12,7 @@ func NoResultFuncHandler(ctx context.Context, flow kis.Flow) error {
fmt.Println("---> Call NoResultFuncHandler ----") fmt.Println("---> Call NoResultFuncHandler ----")
for _, row := range flow.Input() { for _, row := range flow.Input() {
str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetId(), row) str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row)
fmt.Println(str) fmt.Println(str)
} }

View File

@ -17,14 +17,15 @@ type AvgStuScoreOut struct {
proto.StuAvgScore proto.StuAvgScore
} }
// AvgStuScore(FaaS) 计算学生平均分 // AvgStuScore(FaaS) calculates the average score of students
func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error {
for _, row := range rows { for _, row := range rows {
avgScore := proto.StuAvgScore{ avgScore := proto.StuAvgScore{
StuId: row.StuId, StuId: row.StuId,
AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3,
} }
// 提交结果数据
// Submit result data
_ = flow.CommitRow(AvgStuScoreOut{StuAvgScore: avgScore}) _ = flow.CommitRow(AvgStuScoreOut{StuAvgScore: avgScore})
} }

View File

@ -8,18 +8,17 @@ import (
) )
func init() { func init() {
// 0. 注册Function 回调业务 // Register Function callback business
kis.Pool().FaaS("funcName1", faas.FuncDemo1Handler) kis.Pool().FaaS("funcName1", faas.FuncDemo1Handler)
kis.Pool().FaaS("funcName2", faas.FuncDemo2Handler) kis.Pool().FaaS("funcName2", faas.FuncDemo2Handler)
kis.Pool().FaaS("funcName3", faas.FuncDemo3Handler) kis.Pool().FaaS("funcName3", faas.FuncDemo3Handler)
kis.Pool().FaaS("funcName4", faas.FuncDemo4Handler) kis.Pool().FaaS("funcName4", faas.FuncDemo4Handler)
kis.Pool().FaaS("abortFunc", faas.AbortFuncHandler) // abortFunc 业务 kis.Pool().FaaS("abortFunc", faas.AbortFuncHandler) // abortFunc business
kis.Pool().FaaS("dataReuseFunc", faas.DataReuseFuncHandler) // dataReuseFunc 业务 kis.Pool().FaaS("dataReuseFunc", faas.DataReuseFuncHandler) // dataReuseFunc business
kis.Pool().FaaS("noResultFunc", faas.NoResultFuncHandler) // noResultFunc 业务 kis.Pool().FaaS("noResultFunc", faas.NoResultFuncHandler) // noResultFunc business
kis.Pool().FaaS("jumpFunc", faas.JumpFuncHandler) // jumpFunc 业务 kis.Pool().FaaS("jumpFunc", faas.JumpFuncHandler) // jumpFunc business
// 0. 注册ConnectorInit 和 Connector 回调业务 // Register ConnectorInit and Connector callback business
kis.Pool().CaaSInit("ConnName1", caas.InitConnDemo1) kis.Pool().CaaSInit("ConnName1", caas.InitConnDemo1)
kis.Pool().CaaS("ConnName1", "funcName2", common.S, caas.CaasDemoHanler1) kis.Pool().CaaS("ConnName1", "funcName2", common.S, caas.CaasDemoHanler1)
} }

View File

@ -11,21 +11,21 @@ import (
func TestActionAbort(t *testing.T) { func TestActionAbort(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName2") flow1 := kis.Pool().GetFlow("flowName2")
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -34,21 +34,21 @@ func TestActionAbort(t *testing.T) {
func TestActionDataReuse(t *testing.T) { func TestActionDataReuse(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName3") flow1 := kis.Pool().GetFlow("flowName3")
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -57,21 +57,21 @@ func TestActionDataReuse(t *testing.T) {
func TestActionForceEntry(t *testing.T) { func TestActionForceEntry(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName4") flow1 := kis.Pool().GetFlow("flowName4")
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -80,21 +80,21 @@ func TestActionForceEntry(t *testing.T) {
func TestActionJumpFunc(t *testing.T) { func TestActionJumpFunc(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName5") flow1 := kis.Pool().GetFlow("flowName5")
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -18,18 +18,18 @@ func TestAutoInjectParamWithConfig(t *testing.T) {
kis.Pool().FaaS("AvgStuScore", faas.AvgStuScore) kis.Pool().FaaS("AvgStuScore", faas.AvgStuScore)
kis.Pool().FaaS("PrintStuAvgScore", faas.PrintStuAvgScore) kis.Pool().FaaS("PrintStuAvgScore", faas.PrintStuAvgScore)
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("StuAvg") flow1 := kis.Pool().GetFlow("StuAvg")
if flow1 == nil { if flow1 == nil {
panic("flow1 is nil") panic("flow1 is nil")
} }
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow(&faas.AvgStuScoreIn{ _ = flow1.CommitRow(&faas.AvgStuScoreIn{
StuScores: proto.StuScores{ StuScores: proto.StuScores{
StuId: 100, StuId: 100,
@ -47,10 +47,10 @@ func TestAutoInjectParamWithConfig(t *testing.T) {
}, },
}) })
// 提交原始数据json字符串 // Commit original data (as JSON string)
_ = flow1.CommitRow(`{"stu_id":101}`) _ = flow1.CommitRow(`{"stu_id":101}`)
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -81,7 +81,7 @@ func TestAutoInjectParam(t *testing.T) {
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 4. 拼接Functioin 到 Flow 上 // 4. Link Functions to Flow
if err := flow1.Link(avgStuScoreConfig, nil); err != nil { if err := flow1.Link(avgStuScoreConfig, nil); err != nil {
panic(err) panic(err)
} }
@ -89,7 +89,7 @@ func TestAutoInjectParam(t *testing.T) {
panic(err) panic(err)
} }
// 3. 提交原始数据 // 3. Commit original data
_ = flow1.CommitRow(&faas.AvgStuScoreIn{ _ = flow1.CommitRow(&faas.AvgStuScoreIn{
StuScores: proto.StuScores{ StuScores: proto.StuScores{
StuId: 100, StuId: 100,
@ -108,7 +108,7 @@ func TestAutoInjectParam(t *testing.T) {
}, },
}) })
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -7,15 +7,15 @@ import (
"testing" "testing"
) )
func TestConfigExportYmal(t *testing.T) { func TestConfigExportYaml(t *testing.T) {
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 讲构建的内存KisFlow结构配置导出的文件当中 // 2. Export the built memory KisFlow structure configuration to files
flows := kis.Pool().GetFlows() flows := kis.Pool().GetFlows()
for _, flow := range flows { for _, flow := range flows {
if err := file.ConfigExportYaml(flow, "/Users/Aceld/go/src/kis-flow/test/export_conf/"); err != nil { if err := file.ConfigExportYaml(flow, "/Users/Aceld/go/src/kis-flow/test/export_conf/"); err != nil {

View File

@ -10,20 +10,20 @@ import (
func TestConfigImportYaml(t *testing.T) { func TestConfigImportYaml(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName1") flow1 := kis.Pool().GetFlow("flowName1")
// 3. 提交原始数据 // 3. Commit the raw data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -9,7 +9,7 @@ import (
func TestNewFuncConfig(t *testing.T) { func TestNewFuncConfig(t *testing.T) {
source := config.KisSource{ source := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -56,7 +56,7 @@ func TestNewFlowConfig(t *testing.T) {
func TestNewConnConfig(t *testing.T) { func TestNewConnConfig(t *testing.T) {
source := config.KisSource{ source := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }

View File

@ -12,14 +12,14 @@ func TestNewKisConnector(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 创建3个KisFunction配置实例, 其中myFuncConfig2 有Connector配置 // 1. Create three KisFunction configuration instances, with myFuncConfig2 having a Connector configuration
source1 := config.KisSource{ source1 := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
source2 := config.KisSource{ source2 := config.KisSource{
Name: "用户订单错误率", Name: "UserOrderErrorRate",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -42,22 +42,22 @@ func TestNewKisConnector(t *testing.T) {
panic("myFuncConfig3 is nil") panic("myFuncConfig3 is nil")
} }
// 2. 创建一个KisConnector配置实例 // 2. Create a KisConnector configuration instance
myConnConfig1 := config.NewConnConfig("ConnName1", "0.0.0.0:9998", common.REDIS, "redis-key", nil) myConnConfig1 := config.NewConnConfig("ConnName1", "0.0.0.0:9998", common.REDIS, "redis-key", nil)
if myConnConfig1 == nil { if myConnConfig1 == nil {
panic("myConnConfig1 is nil") panic("myConnConfig1 is nil")
} }
// 3. 将KisConnector配置实例绑定到KisFunction配置实例上 // 3. Bind the KisConnector configuration instance to the KisFunction configuration instance
_ = myFuncConfig2.AddConnConfig(myConnConfig1) _ = myFuncConfig2.AddConnConfig(myConnConfig1)
// 4. 创建一个 KisFlow 配置实例 // 4. Create a KisFlow configuration instance
myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable)
// 5. 创建一个KisFlow对象 // 5. Create a KisFlow object
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 6. 拼接Functioin 到 Flow 上 // 6. Link Functions to the Flow
if err := flow1.Link(myFuncConfig1, nil); err != nil { if err := flow1.Link(myFuncConfig1, nil); err != nil {
panic(err) panic(err)
} }
@ -68,12 +68,12 @@ func TestNewKisConnector(t *testing.T) {
panic(err) panic(err)
} }
// 7. 提交原始数据 // 7. Commit raw data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 8. 执行flow1 // 8. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -11,12 +11,12 @@ import (
func TestForkFlowCommitBatch(t *testing.T) { func TestForkFlowCommitBatch(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load the configuration file and build the Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get the Flow
flow1 := kis.Pool().GetFlow("flowName1") flow1 := kis.Pool().GetFlow("flowName1")
stringRows := []string{ stringRows := []string{
@ -25,13 +25,13 @@ func TestForkFlowCommitBatch(t *testing.T) {
"This is Data3 from Test", "This is Data3 from Test",
} }
// 3. 提交原始数据 // 3. Commit raw data
if err := flow1.CommitRowBatch(stringRows); err != nil { if err := flow1.CommitRowBatch(stringRows); err != nil {
log.Logger().ErrorF("CommitRowBatch Error, err = %+v", err) log.Logger().ErrorF("CommitRowBatch Error, err = %+v", err)
panic(err) panic(err)
} }
// 4. 执行flow1 // 4. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -11,14 +11,14 @@ import (
func TestNewKisFlow(t *testing.T) { func TestNewKisFlow(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 创建2个KisFunction配置实例 // 1. Create 2 KisFunction configuration instances
source1 := config.KisSource{ source1 := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
source2 := config.KisSource{ source2 := config.KisSource{
Name: "用户订单错误率", Name: "UserOrderErrorRate",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -32,13 +32,13 @@ func TestNewKisFlow(t *testing.T) {
panic("myFuncConfig2 is nil") panic("myFuncConfig2 is nil")
} }
// 2. 创建一个 KisFlow 配置实例 // 2. Create a KisFlow configuration instance
myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable)
// 3. 创建一个KisFlow对象 // 3. Create a KisFlow object
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 4. 拼接Functioin 到 Flow 上 // 4. Link functions to the Flow
if err := flow1.Link(myFuncConfig1, nil); err != nil { if err := flow1.Link(myFuncConfig1, nil); err != nil {
panic(err) panic(err)
} }
@ -46,7 +46,7 @@ func TestNewKisFlow(t *testing.T) {
panic(err) panic(err)
} }
// 5. 执行flow1 // 5. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -55,14 +55,14 @@ func TestNewKisFlow(t *testing.T) {
func TestNewKisFlowData(t *testing.T) { func TestNewKisFlowData(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 创建2个KisFunction配置实例 // 1. Create 2 KisFunction configuration instances
source1 := config.KisSource{ source1 := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
source2 := config.KisSource{ source2 := config.KisSource{
Name: "用户订单错误率", Name: "UserOrderErrorRate",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -76,13 +76,13 @@ func TestNewKisFlowData(t *testing.T) {
panic("myFuncConfig4 is nil") panic("myFuncConfig4 is nil")
} }
// 2. 创建一个 KisFlow 配置实例 // 2. Create a KisFlow configuration instance
myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable)
// 3. 创建一个KisFlow对象 // 3. Create a KisFlow object
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 4. 拼接 Function 到 Flow 上 // 4. Link Function to the Flow
if err := flow1.Link(myFuncConfig1, nil); err != nil { if err := flow1.Link(myFuncConfig1, nil); err != nil {
panic(err) panic(err)
} }
@ -90,12 +90,12 @@ func TestNewKisFlowData(t *testing.T) {
panic(err) panic(err)
} }
// 5. 提交原始数据 // 5. Commit raw data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 6. 执行flow1 // 6. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -14,12 +14,12 @@ import (
func TestForkFlow(t *testing.T) { func TestForkFlow(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow // 1. Load configuration file and build Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
panic(err) panic(err)
} }
// 2. 获取Flow // 2. Get Flow
flow1 := kis.Pool().GetFlow("flowFork1") flow1 := kis.Pool().GetFlow("flowFork1")
fmt.Println("----> flow1: ", flow1.GetFuncParamsAllFuncs()) fmt.Println("----> flow1: ", flow1.GetFuncParamsAllFuncs())
@ -27,10 +27,10 @@ func TestForkFlow(t *testing.T) {
flow1Clone1 := flow1.Fork(ctx) flow1Clone1 := flow1.Fork(ctx)
fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs()) fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs())
// 3. 提交原始数据 // 3. Commit raw data
_ = flow1Clone1.CommitRow("This is Data1 from Test") _ = flow1Clone1.CommitRow("This is Data1 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1Clone1.Run(ctx); err != nil { if err := flow1Clone1.Run(ctx); err != nil {
panic(err) panic(err)
} }
@ -58,10 +58,10 @@ func TestForkFlowWithLink(t *testing.T) {
fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs()) fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs())
// 3. 提交原始数据 // 3. Commit raw data
_ = flow1Clone1.CommitRow("This is Data1 from Test") _ = flow1Clone1.CommitRow("This is Data1 from Test")
// 4. 执行flow1 // 4. Execute flow1
if err := flow1Clone1.Run(ctx); err != nil { if err := flow1Clone1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -12,9 +12,9 @@ import (
func TestNewKisFunction(t *testing.T) { func TestNewKisFunction(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 创建一个KisFunction配置实例 // 1. Create a KisFunction configuration instance
source := config.KisSource{ source := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TikTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -23,13 +23,13 @@ func TestNewKisFunction(t *testing.T) {
panic("myFuncConfig1 is nil") panic("myFuncConfig1 is nil")
} }
// 2. 创建一个 KisFlow 配置实例 // 2. Create a KisFlow configuration instance
myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable)
// 3. 创建一个KisFlow对象 // 3. Create a KisFlow object
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 4. 创建一个KisFunction对象 // 4. Create a KisFunction object
func1 := function.NewKisFunction(flow1, myFuncConfig1) func1 := function.NewKisFunction(flow1, myFuncConfig1)
if err := func1.Call(ctx, flow1); err != nil { if err := func1.Call(ctx, flow1); err != nil {

View File

@ -12,22 +12,18 @@ import (
func TestMetricsDataTotal(t *testing.T) { func TestMetricsDataTotal(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
fmt.Println("Wrong Config Yaml Path!") fmt.Println("Wrong Config Yaml Path!")
panic(err) panic(err)
} }
// 2. 获取Flow
flow1 := kis.Pool().GetFlow("flowName1") flow1 := kis.Pool().GetFlow("flowName1")
n := 0 n := 0
for n < 10 { for n < 10 {
// 3. 提交原始数据
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
// 4. 执行flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -10,20 +10,16 @@ import (
func TestParams(t *testing.T) { func TestParams(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 加载配置文件并构建Flow
if err := file.ConfigImportYaml("load_conf/"); err != nil { if err := file.ConfigImportYaml("load_conf/"); err != nil {
panic(err) panic(err)
} }
// 2. 获取Flow
flow1 := kis.Pool().GetFlow("flowName1") flow1 := kis.Pool().GetFlow("flowName1")
// 3. 提交原始数据
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 4. 执行flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -12,14 +12,14 @@ func TestNewKisPool(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// 1. 创建2个KisFunction配置实例 // 1. Create 2 KisFunction configuration instances
source1 := config.KisSource{ source1 := config.KisSource{
Name: "公众号抖音商城户订单数据", Name: "TickTokOrder",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
source2 := config.KisSource{ source2 := config.KisSource{
Name: "用户订单错误率", Name: "UserOrderErrorRate",
Must: []string{"order_id", "user_id"}, Must: []string{"order_id", "user_id"},
} }
@ -33,13 +33,13 @@ func TestNewKisPool(t *testing.T) {
panic("myFuncConfig4 is nil") panic("myFuncConfig4 is nil")
} }
// 2. 创建一个 KisFlow 配置实例 // 2. Create a KisFlow configuration instance
myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable)
// 3. 创建一个KisFlow对象 // 3. Create a KisFlow object
flow1 := flow.NewKisFlow(myFlowConfig1) flow1 := flow.NewKisFlow(myFlowConfig1)
// 4. 拼接Functioin 到 Flow 上 // 4. Link Functions to Flow
if err := flow1.Link(myFuncConfig1, nil); err != nil { if err := flow1.Link(myFuncConfig1, nil); err != nil {
panic(err) panic(err)
} }
@ -47,12 +47,12 @@ func TestNewKisPool(t *testing.T) {
panic(err) panic(err)
} }
// 5. 提交原始数据 // 5. Commit raw data
_ = flow1.CommitRow("This is Data1 from Test") _ = flow1.CommitRow("This is Data1 from Test")
_ = flow1.CommitRow("This is Data2 from Test") _ = flow1.CommitRow("This is Data2 from Test")
_ = flow1.CommitRow("This is Data3 from Test") _ = flow1.CommitRow("This is Data3 from Test")
// 6. 执行flow1 // 6. Execute flow1
if err := flow1.Run(ctx); err != nil { if err := flow1.Run(ctx); err != nil {
panic(err) panic(err)
} }

View File

@ -2,7 +2,7 @@ kistype: func
fname: abortFunc fname: abortFunc
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,6 +2,6 @@ kistype: func
fname: AvgStuScore fname: AvgStuScore
fmode: Calculate fmode: Calculate
source: source:
name: 学生平均分 name: StudentAverageScore
must: must:
- stu_id - stu_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName1 fname: funcName1
fmode: Verify fmode: Verify
source: source:
name: 公众号抖音商城户订单数据 name: TiktokOrder
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName2 fname: funcName2
fmode: Save fmode: Save
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: funcName3 fname: funcName3
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: noResultFunc fname: noResultFunc
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,6 +2,6 @@ kistype: func
fname: PrintStuAvgScore fname: PrintStuAvgScore
fmode: Expand fmode: Expand
source: source:
name: 学生平均分 name: StudentAverageScore
must: must:
- stu_id - stu_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: dataReuseFunc fname: dataReuseFunc
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -2,7 +2,7 @@ kistype: func
fname: jumpFunc fname: jumpFunc
fmode: Calculate fmode: Calculate
source: source:
name: 用户订单错误率 name: UserOrderErrorRate
must: must:
- order_id - order_id
- user_id - user_id

View File

@ -1,8 +1,8 @@
#kistype Global为kisflow的全局配置 # kistype Global is the global configuration for kisflow
kistype: global kistype: global
#是否启动prometheus监控 # Whether to enable prometheus monitoring
prometheus_enable: true prometheus_enable: true
#是否需要kisflow单独启动端口监听 # Whether kisflow needs to listen on a separate port
prometheus_listen: true prometheus_listen: true
#prometheus取点监听地址 # Prometheus scrape address
prometheus_serve: 0.0.0.0:20004 prometheus_serve: 0.0.0.0:20004