Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
b62b1786
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
b62b1786
编写于
4月 19, 2022
作者:
S
shenglian-zhou
提交者:
GitHub
4月 19, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #11572 from taosdata/3.0_udfd
feat(udf): UDF service refactoring with new interface
上级
e5bf165e
adf99575
变更
7
展开全部
隐藏空白更改
内联
并排
Showing
7 changed file
with
1019 addition
and
570 deletion
+1019
-570
source/libs/function/CMakeLists.txt
source/libs/function/CMakeLists.txt
+13
-0
source/libs/function/inc/tudf.h
source/libs/function/inc/tudf.h
+83
-65
source/libs/function/inc/tudfInt.h
source/libs/function/inc/tudfInt.h
+40
-24
source/libs/function/src/tudf.c
source/libs/function/src/tudf.c
+639
-373
source/libs/function/src/udfd.c
source/libs/function/src/udfd.c
+153
-76
source/libs/function/test/runUdf.c
source/libs/function/test/runUdf.c
+29
-18
source/libs/function/test/udf1.c
source/libs/function/test/udf1.c
+62
-14
未找到文件。
source/libs/function/CMakeLists.txt
浏览文件 @
b62b1786
...
...
@@ -5,6 +5,9 @@ target_include_directories(
function
PUBLIC
"
${
TD_SOURCE_DIR
}
/include/libs/function"
"
${
TD_SOURCE_DIR
}
/include/util"
"
${
TD_SOURCE_DIR
}
/include/common"
"
${
TD_SOURCE_DIR
}
/include/client"
"
${
TD_SOURCE_DIR
}
/contrib/libuv/include"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
...
...
@@ -21,6 +24,9 @@ target_include_directories(
PUBLIC
"
${
TD_SOURCE_DIR
}
/include/libs/function"
"
${
TD_SOURCE_DIR
}
/contrib/libuv/include"
"
${
TD_SOURCE_DIR
}
/include/util"
"
${
TD_SOURCE_DIR
}
/include/common"
"
${
TD_SOURCE_DIR
}
/include/client"
"
${
TD_SOURCE_DIR
}
/include/os"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
...
...
@@ -35,6 +41,9 @@ target_include_directories(
udf1
PUBLIC
"
${
TD_SOURCE_DIR
}
/include/libs/function"
"
${
TD_SOURCE_DIR
}
/include/util"
"
${
TD_SOURCE_DIR
}
/include/common"
"
${
TD_SOURCE_DIR
}
/include/client"
"
${
TD_SOURCE_DIR
}
/include/os"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
...
...
@@ -46,6 +55,10 @@ target_include_directories(
PUBLIC
"
${
TD_SOURCE_DIR
}
/include/libs/function"
"
${
TD_SOURCE_DIR
}
/contrib/libuv/include"
"
${
TD_SOURCE_DIR
}
/include/util"
"
${
TD_SOURCE_DIR
}
/include/common"
"
${
TD_SOURCE_DIR
}
/include/libs/transport"
"
${
TD_SOURCE_DIR
}
/include/client"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
...
...
source/libs/function/inc/tudf.h
浏览文件 @
b62b1786
...
...
@@ -16,12 +16,25 @@
#ifndef TDENGINE_TUDF_H
#define TDENGINE_TUDF_H
#include <stdint.h>
#include <stdbool.h>
#include "tmsg.h"
#include "tcommon.h"
#ifdef __cplusplus
extern
"C"
{
#endif
//======================================================================================
//begin API to taosd and qworker
enum
{
UDFC_CODE_STOPPING
=
-
1
,
UDFC_CODE_RESTARTING
=
-
2
,
UDFC_CODE_PIPE_READ_ERR
=
-
3
,
};
/**
* start udf dameon service
* @return error code
...
...
@@ -34,28 +47,6 @@ int32_t startUdfService();
*/
int32_t
stopUdfService
();
enum
{
TSDB_UDF_TYPE_SCALAR
=
0
,
TSDB_UDF_TYPE_AGGREGATE
=
1
};
enum
{
TSDB_UDF_SCRIPT_BIN_LIB
=
0
,
TSDB_UDF_SCRIPT_LUA
=
1
,
};
typedef
struct
SUdfInfo
{
char
*
udfName
;
// function name
int32_t
udfType
;
// scalar function or aggregate function
int8_t
scriptType
;
char
*
path
;
int8_t
resType
;
// result type
int16_t
resBytes
;
// result byte
int32_t
bufSize
;
//interbuf size
}
SUdfInfo
;
typedef
void
*
UdfHandle
;
/**
...
...
@@ -64,36 +55,67 @@ typedef void *UdfHandle;
* @param handle, out
* @return error code
*/
int32_t
setupUdf
(
SUdfInfo
*
udf
,
UdfHandle
*
handle
);
enum
{
TSDB_UDF_STEP_NORMAL
=
0
,
TSDB_UDF_STEP_MERGE
,
TSDb_UDF_STEP_FINALIZE
,
TSDB_UDF_STEP_MAX_NUM
};
/**
* call udf
* @param handle udf handle
* @param step
* @param state
* @param stateSize
* @param input
* @param newstate
* @param newStateSize
* @param output
* @return error code
*/
int32_t
setupUdf
(
char
udfName
[],
SEpSet
*
epSet
,
UdfHandle
*
handle
);
typedef
struct
SUdfColumnMeta
{
int16_t
type
;
int32_t
bytes
;
// <0 var length, others fixed length bytes
uint8_t
precision
;
uint8_t
scale
;
}
SUdfColumnMeta
;
typedef
struct
SUdfColumnData
{
int32_t
numOfRows
;
bool
varLengthColumn
;
union
{
struct
{
int32_t
nullBitmapLen
;
char
*
nullBitmap
;
int32_t
dataLen
;
char
*
data
;
}
fixLenCol
;
struct
{
int32_t
varOffsetsLen
;
char
*
varOffsets
;
int32_t
payloadLen
;
char
*
payload
;
}
varLenCol
;
};
}
SUdfColumnData
;
typedef
struct
SUdfColumn
{
SUdfColumnMeta
colMeta
;
SUdfColumnData
colData
;
}
SUdfColumn
;
//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined
typedef
struct
SUdfDataBlock
{
char
*
data
;
int32_t
size
;
int32_t
numOfRows
;
int32_t
numOfCols
;
SUdfColumn
**
udfCols
;
}
SUdfDataBlock
;
int32_t
callUdf
(
UdfHandle
handle
,
int8_t
step
,
char
*
state
,
int32_t
stateSize
,
SUdfDataBlock
input
,
char
**
newstate
,
int32_t
*
newStateSize
,
SUdfDataBlock
*
output
);
typedef
struct
SUdfInterBuf
{
int32_t
bufLen
;
char
*
buf
;
}
SUdfInterBuf
;
//TODO: translate these calls to callUdf
// output: interBuf
int32_t
callUdfAggInit
(
UdfHandle
handle
,
SUdfInterBuf
*
interBuf
);
// input: block, state
// output: newState
int32_t
callUdfAggProcess
(
UdfHandle
handle
,
SSDataBlock
*
block
,
SUdfInterBuf
*
state
,
SUdfInterBuf
*
newState
);
// input: interBuf
// output: resultData
int32_t
callUdfAggFinalize
(
UdfHandle
handle
,
SUdfInterBuf
*
interBuf
,
SUdfInterBuf
*
resultData
);
// input: interbuf1, interbuf2
// output: resultBuf
int32_t
callUdfAggMerge
(
UdfHandle
handle
,
SUdfInterBuf
*
interBuf1
,
SUdfInterBuf
*
interBuf2
,
SUdfInterBuf
*
resultBuf
);
// input: block
// output: resultData
int32_t
callUdfScalaProcess
(
UdfHandle
handle
,
SSDataBlock
*
block
,
SSDataBlock
*
resultData
);
/**
* tearn down udf
...
...
@@ -104,29 +126,25 @@ int32_t teardownUdf(UdfHandle handle);
// end API to taosd and qworker
//=============================================================================================================================
// TODO: Must change
// begin API to UDF writer.
// script
// dynamic lib init and destroy
typedef
int32_t
(
*
TUdfSetupFunc
)();
typedef
int32_t
(
*
TUdfTeardownFunc
)();
//typedef int32_t (*scriptInitFunc)(void* pCtx);
//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
// int16_t oType, int16_t oBytes);
//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
//typedef void (*scriptDestroyFunc)(void* pCtx);
//TODO: add API to check function arguments type, number etc.
//TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory.
// then UDF framework will free the memory
//typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data);
//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
// dynamic lib
typedef
int32_t
(
*
TUdfInitFunc
)();
typedef
void
(
*
TUdfDestroyFunc
)();
typedef
int32_t
(
*
TUdfFreeUdfColumnFunc
)(
SUdfColumn
*
column
);
typedef
void
(
*
TUdfFunc
)(
int8_t
step
,
char
*
state
,
int32_t
stateSize
,
SUdfDataBlock
input
,
char
**
newstate
,
int32_t
*
newStateSize
,
SUdfDataBlock
*
output
);
typedef
int32_t
(
*
TUdfScalarProcFunc
)(
SUdfDataBlock
block
,
SUdfColumn
*
resultCol
);
typedef
int32_t
(
*
TUdfAggInitFunc
)(
SUdfInterBuf
*
buf
);
typedef
int32_t
(
*
TUdfAggProcessFunc
)(
SUdfDataBlock
block
,
SUdfInterBuf
*
interBuf
);
typedef
int32_t
(
*
TUdfAggFinalizeFunc
)(
SUdfInterBuf
buf
,
SUdfInterBuf
*
resultData
);
//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput);
//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output);
// end API to UDF writer
//=======================================================================================================================
...
...
source/libs/function/inc/tudfInt.h
浏览文件 @
b62b1786
...
...
@@ -15,7 +15,6 @@
#ifndef TDENGINE_TUDF_INT_H
#define TDENGINE_TUDF_INT_H
#ifdef __cplusplus
extern
"C"
{
#endif
...
...
@@ -30,36 +29,37 @@ enum {
};
enum
{
TSDB_UDF_CALL_AGG_INIT
=
0
,
TSDB_UDF_CALL_AGG_PROC
,
TSDB_UDF_CALL_AGG_MERGE
,
TSDB_UDF_CALL_AGG_FIN
,
TSDB_UDF_CALL_SCALA_PROC
,
};
typedef
struct
SUdfSetupRequest
{
char
udfName
[
16
];
//
int8_t
scriptType
;
// 0:c, 1: lua, 2:js
int8_t
udfType
;
//udaf, udf
int16_t
pathSize
;
char
*
path
;
char
udfName
[
TSDB_FUNC_NAME_LEN
];
SEpSet
epSet
;
}
SUdfSetupRequest
;
typedef
struct
SUdfSetupResponse
{
int64_t
udfHandle
;
}
SUdfSetupResponse
;
typedef
struct
SUdfCallRequest
{
int64_t
udfHandle
;
int8_t
step
;
int8_t
callType
;
int32_t
inputBytes
;
char
*
input
;
int32_t
stateBytes
;
char
*
state
;
SSDataBlock
block
;
SUdfInterBuf
interBuf
;
SUdfInterBuf
interBuf2
;
int8_t
initFirst
;
}
SUdfCallRequest
;
typedef
struct
SUdfCallResponse
{
int32_t
outputBytes
;
char
*
output
;
int32_t
newStateBytes
;
char
*
newState
;
int8_t
callType
;
SSDataBlock
resultData
;
SUdfInterBuf
resultBuf
;
}
SUdfCallResponse
;
...
...
@@ -76,7 +76,11 @@ typedef struct SUdfRequest {
int64_t
seqNum
;
int8_t
type
;
void
*
subReq
;
union
{
SUdfSetupRequest
setup
;
SUdfCallRequest
call
;
SUdfTeardownRequest
teardown
;
};
}
SUdfRequest
;
typedef
struct
SUdfResponse
{
...
...
@@ -85,13 +89,25 @@ typedef struct SUdfResponse {
int8_t
type
;
int32_t
code
;
void
*
subRsp
;
union
{
SUdfSetupResponse
setupRsp
;
SUdfCallResponse
callRsp
;
SUdfTeardownResponse
teardownRsp
;
};
}
SUdfResponse
;
int32_t
decodeRequest
(
char
*
buf
,
int32_t
bufLen
,
SUdfRequest
**
pRequest
);
int32_t
encodeResponse
(
char
**
buf
,
int32_t
*
bufLen
,
SUdfResponse
*
response
);
int32_t
encodeRequest
(
char
**
buf
,
int32_t
*
bufLen
,
SUdfRequest
*
request
);
int32_t
decodeResponse
(
char
*
buf
,
int32_t
bufLen
,
SUdfResponse
**
pResponse
);
int32_t
encodeUdfRequest
(
void
**
buf
,
const
SUdfRequest
*
request
);
void
*
decodeUdfRequest
(
const
void
*
buf
,
SUdfRequest
*
request
);
int32_t
encodeUdfResponse
(
void
**
buf
,
const
SUdfResponse
*
response
);
void
*
decodeUdfResponse
(
const
void
*
buf
,
SUdfResponse
*
response
);
void
freeUdfColumnData
(
SUdfColumnData
*
data
);
void
freeUdfColumn
(
SUdfColumn
*
col
);
void
freeUdfDataDataBlock
(
SUdfDataBlock
*
block
);
int32_t
convertDataBlockToUdfDataBlock
(
SSDataBlock
*
block
,
SUdfDataBlock
*
udfBlock
);
int32_t
convertUdfColumnToDataBlock
(
SUdfColumn
*
udfCol
,
SSDataBlock
*
block
);
#ifdef __cplusplus
}
...
...
source/libs/function/src/tudf.c
浏览文件 @
b62b1786
此差异已折叠。
点击以展开。
source/libs/function/src/udfd.c
浏览文件 @
b62b1786
...
...
@@ -20,6 +20,10 @@
#include "tudf.h"
#include "tudfInt.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"
#include "trpc.h"
static
uv_loop_t
*
loop
;
...
...
@@ -44,7 +48,8 @@ typedef struct SUdf {
int8_t
type
;
uv_lib_t
lib
;
TUdfFunc
normalFunc
;
TUdfScalarProcFunc
scalarProcFunc
;
TUdfFreeUdfColumnFunc
freeUdfColumn
;
}
SUdf
;
//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
...
...
@@ -56,119 +61,120 @@ typedef struct SUdfHandle {
void
udfdProcessRequest
(
uv_work_t
*
req
)
{
SUvUdfWork
*
uvUdf
=
(
SUvUdfWork
*
)
(
req
->
data
);
SUdfRequest
*
request
=
NULL
;
decode
Request
(
uvUdf
->
input
.
base
,
uvUdf
->
input
.
len
,
&
request
);
SUdfRequest
request
=
{
0
}
;
decode
UdfRequest
(
uvUdf
->
input
.
base
,
&
request
);
switch
(
request
->
type
)
{
switch
(
request
.
type
)
{
case
UDF_TASK_SETUP
:
{
debugPrint
(
"%s"
,
"process setup request"
);
SUdf
*
udf
=
taosMemoryMalloc
(
sizeof
(
SUdf
));
udf
->
refCount
=
0
;
SUdfSetupRequest
*
setup
=
request
->
subReq
;
SUdfSetupRequest
*
setup
=
&
request
.
setup
;
strcpy
(
udf
->
name
,
setup
->
udfName
);
int
err
=
uv_dlopen
(
setup
->
path
,
&
udf
->
lib
);
//TODO: retrive udf info from mnode
char
*
path
=
"libudf1.so"
;
int
err
=
uv_dlopen
(
path
,
&
udf
->
lib
);
if
(
err
!=
0
)
{
debugPrint
(
"can not load library %s. error: %s"
,
setup
->
path
,
uv_strerror
(
err
));
debugPrint
(
"can not load library %s. error: %s"
,
path
,
uv_strerror
(
err
));
//TODO set error
}
char
normalFuncName
[
32
]
=
{
0
};
char
normalFuncName
[
TSDB_FUNC_NAME_LEN
]
=
{
0
};
strcpy
(
normalFuncName
,
setup
->
udfName
);
//TODO error,
//TODO error,
multi-thread, same udf, lock it
//TODO find all functions normal, init, destroy, normal, merge, finalize
uv_dlsym
(
&
udf
->
lib
,
normalFuncName
,
(
void
**
)
(
&
udf
->
normalFunc
));
uv_dlsym
(
&
udf
->
lib
,
normalFuncName
,
(
void
**
)
(
&
udf
->
scalarProcFunc
));
char
freeFuncName
[
TSDB_FUNC_NAME_LEN
+
6
]
=
{
0
};
char
*
freeSuffix
=
"_free"
;
strncpy
(
freeFuncName
,
normalFuncName
,
strlen
(
normalFuncName
));
strncat
(
freeFuncName
,
freeSuffix
,
strlen
(
freeSuffix
));
uv_dlsym
(
&
udf
->
lib
,
freeFuncName
,
(
void
**
)(
&
udf
->
freeUdfColumn
));
SUdfHandle
*
handle
=
taosMemoryMalloc
(
sizeof
(
SUdfHandle
));
handle
->
udf
=
udf
;
udf
->
refCount
++
;
//TODO: allocate private structure and call init function and set it to handle
SUdfResponse
*
rsp
=
taosMemoryMalloc
(
sizeof
(
SUdfResponse
));
rsp
->
seqNum
=
request
->
seqNum
;
rsp
->
type
=
request
->
type
;
rsp
->
code
=
0
;
SUdfSetupResponse
*
subRsp
=
taosMemoryMalloc
(
sizeof
(
SUdfSetupResponse
));
subRsp
->
udfHandle
=
(
int64_t
)
(
handle
);
rsp
->
subRsp
=
subRsp
;
char
*
buf
;
int32_t
len
;
encodeResponse
(
&
buf
,
&
len
,
rsp
);
uvUdf
->
output
=
uv_buf_init
(
buf
,
len
);
taosMemoryFree
(
rsp
->
subRsp
);
taosMemoryFree
(
rsp
);
taosMemoryFree
(
request
->
subReq
);
taosMemoryFree
(
request
);
SUdfResponse
rsp
;
rsp
.
seqNum
=
request
.
seqNum
;
rsp
.
type
=
request
.
type
;
rsp
.
code
=
0
;
rsp
.
setupRsp
.
udfHandle
=
(
int64_t
)
(
handle
);
int32_t
len
=
encodeUdfResponse
(
NULL
,
&
rsp
);
rsp
.
msgLen
=
len
;
void
*
bufBegin
=
taosMemoryMalloc
(
len
);
void
*
buf
=
bufBegin
;
encodeUdfResponse
(
&
buf
,
&
rsp
);
uvUdf
->
output
=
uv_buf_init
(
bufBegin
,
len
);
taosMemoryFree
(
uvUdf
->
input
.
base
);
break
;
}
case
UDF_TASK_CALL
:
{
debugPrint
(
"%s"
,
"process call request"
);
SUdfCallRequest
*
call
=
request
->
subReq
;
SUdfCallRequest
*
call
=
&
request
.
call
;
SUdfHandle
*
handle
=
(
SUdfHandle
*
)
(
call
->
udfHandle
);
SUdf
*
udf
=
handle
->
udf
;
char
*
newState
;
int32_t
newStateSize
;
SUdfDataBlock
input
=
{.
data
=
call
->
input
,
.
size
=
call
->
inputBytes
};
SUdfDataBlock
output
;
//TODO: call different functions according to the step
udf
->
normalFunc
(
call
->
step
,
call
->
state
,
call
->
stateBytes
,
input
,
&
newState
,
&
newStateSize
,
&
output
);
SUdfResponse
*
rsp
=
taosMemoryMalloc
(
sizeof
(
SUdfResponse
));
rsp
->
seqNum
=
request
->
seqNum
;
rsp
->
type
=
request
->
type
;
rsp
->
code
=
0
;
SUdfCallResponse
*
subRsp
=
taosMemoryMalloc
(
sizeof
(
SUdfCallResponse
));
subRsp
->
outputBytes
=
output
.
size
;
subRsp
->
output
=
output
.
data
;
subRsp
->
newStateBytes
=
newStateSize
;
subRsp
->
newState
=
newState
;
rsp
->
subRsp
=
subRsp
;
char
*
buf
;
int32_t
len
;
encodeResponse
(
&
buf
,
&
len
,
rsp
);
uvUdf
->
output
=
uv_buf_init
(
buf
,
len
);
taosMemoryFree
(
rsp
->
subRsp
);
taosMemoryFree
(
rsp
);
taosMemoryFree
(
newState
);
taosMemoryFree
(
output
.
data
);
taosMemoryFree
(
request
->
subReq
);
taosMemoryFree
(
request
);
SUdfDataBlock
input
=
{
0
};
convertDataBlockToUdfDataBlock
(
&
call
->
block
,
&
input
);
SUdfColumn
output
=
{
0
};
//TODO: call different functions according to call type, for now just calar
if
(
call
->
callType
==
TSDB_UDF_CALL_SCALA_PROC
)
{
udf
->
scalarProcFunc
(
input
,
&
output
);
}
SUdfResponse
response
=
{
0
};
SUdfResponse
*
rsp
=
&
response
;
if
(
call
->
callType
==
TSDB_UDF_CALL_SCALA_PROC
)
{
rsp
->
seqNum
=
request
.
seqNum
;
rsp
->
type
=
request
.
type
;
rsp
->
code
=
0
;
SUdfCallResponse
*
subRsp
=
&
rsp
->
callRsp
;
subRsp
->
callType
=
call
->
callType
;
convertUdfColumnToDataBlock
(
&
output
,
&
subRsp
->
resultData
);
}
int32_t
len
=
encodeUdfResponse
(
NULL
,
rsp
);
rsp
->
msgLen
=
len
;
void
*
bufBegin
=
taosMemoryMalloc
(
len
);
void
*
buf
=
bufBegin
;
encodeUdfResponse
(
&
buf
,
rsp
);
uvUdf
->
output
=
uv_buf_init
(
bufBegin
,
len
);
//TODO: free
udf
->
freeUdfColumn
(
&
output
);
taosMemoryFree
(
uvUdf
->
input
.
base
);
break
;
}
case
UDF_TASK_TEARDOWN
:
{
debugPrint
(
"%s"
,
"process teardown request"
);
SUdfTeardownRequest
*
teardown
=
request
->
subReq
;
SUdfTeardownRequest
*
teardown
=
&
request
.
teardown
;
SUdfHandle
*
handle
=
(
SUdfHandle
*
)
(
teardown
->
udfHandle
);
SUdf
*
udf
=
handle
->
udf
;
udf
->
refCount
--
;
if
(
udf
->
refCount
==
0
)
{
uv_dlclose
(
&
udf
->
lib
);
taosMemoryFree
(
udf
);
}
taosMemoryFree
(
udf
);
//TODO: call destroy and free udf private
//TODO: call destroy and free udf private
taosMemoryFree
(
handle
);
SUdfResponse
*
rsp
=
taosMemoryMalloc
(
sizeof
(
SUdfResponse
));
rsp
->
seqNum
=
request
->
seqNum
;
rsp
->
type
=
request
->
type
;
SUdfResponse
response
;
SUdfResponse
*
rsp
=
&
response
;
rsp
->
seqNum
=
request
.
seqNum
;
rsp
->
type
=
request
.
type
;
rsp
->
code
=
0
;
SUdfTeardownResponse
*
subRsp
=
taosMemoryMalloc
(
sizeof
(
SUdfTeardownResponse
));
rsp
->
subRsp
=
subRsp
;
char
*
buf
;
int32_t
len
;
encodeResponse
(
&
buf
,
&
len
,
rsp
);
uvUdf
->
output
=
uv_buf_init
(
buf
,
len
);
taosMemoryFree
(
rsp
->
subRsp
);
taosMemoryFree
(
rsp
);
taosMemoryFree
(
request
->
subReq
);
taosMemoryFree
(
request
);
int32_t
len
=
encodeUdfResponse
(
NULL
,
rsp
);
rsp
->
msgLen
=
len
;
void
*
bufBegin
=
taosMemoryMalloc
(
len
);
void
*
buf
=
bufBegin
;
encodeUdfResponse
(
&
buf
,
rsp
);
uvUdf
->
output
=
uv_buf_init
(
bufBegin
,
len
);
taosMemoryFree
(
uvUdf
->
input
.
base
);
break
;
}
...
...
@@ -181,7 +187,7 @@ void udfdProcessRequest(uv_work_t *req) {
}
void
udfdOnWrite
(
uv_write_t
*
req
,
int
status
)
{
debugPrint
(
"%s"
,
"after writing to pipe"
);
debugPrint
(
"%s"
,
"
server
after writing to pipe"
);
if
(
status
<
0
)
{
debugPrint
(
"Write error %s"
,
uv_err_name
(
status
));
}
...
...
@@ -205,7 +211,7 @@ void udfdSendResponse(uv_work_t *work, int status) {
}
void
udfdAllocBuffer
(
uv_handle_t
*
handle
,
size_t
suggestedSize
,
uv_buf_t
*
buf
)
{
debugPrint
(
"%s"
,
"allocate buffer for read"
);
debugPrint
(
"%s"
,
"
server
allocate buffer for read"
);
SUdfdUvConn
*
ctx
=
handle
->
data
;
int32_t
msgHeadSize
=
sizeof
(
int32_t
)
+
sizeof
(
int64_t
);
if
(
ctx
->
inputCap
==
0
)
{
...
...
@@ -329,6 +335,77 @@ void removeListeningPipe(int sig) {
exit
(
0
);
}
typedef
struct
SServerContext
{
void
*
clientRpc
;
}
SUdfdContext
;
void
udfdProcessRpcRsp
(
void
*
parent
,
SRpcMsg
*
pMsg
,
SEpSet
*
pEpSet
)
{
return
;
}
int32_t
fetchUdfFuncInfo
(
void
*
clientRpc
,
SEpSet
*
pEpSet
,
char
*
udfNames
[],
int32_t
numOfUdfs
)
{
SRetrieveFuncReq
retrieveReq
=
{
0
};
retrieveReq
.
numOfFuncs
=
1
;
retrieveReq
.
pFuncNames
=
taosArrayInit
(
1
,
TSDB_FUNC_NAME_LEN
);
for
(
int32_t
i
=
0
;
i
<
numOfUdfs
;
++
i
)
{
taosArrayPush
(
retrieveReq
.
pFuncNames
,
udfNames
[
i
]);
}
int32_t
contLen
=
tSerializeSRetrieveFuncReq
(
NULL
,
0
,
&
retrieveReq
);
void
*
pReq
=
rpcMallocCont
(
contLen
);
tSerializeSRetrieveFuncReq
(
pReq
,
contLen
,
&
retrieveReq
);
taosArrayDestroy
(
retrieveReq
.
pFuncNames
);
SRpcMsg
rpcMsg
=
{
0
};
rpcMsg
.
pCont
=
pReq
;
rpcMsg
.
contLen
=
contLen
;
rpcMsg
.
msgType
=
TDMT_MND_RETRIEVE_FUNC
;
SRpcMsg
rpcRsp
=
{
0
};
rpcSendRecv
(
clientRpc
,
pEpSet
,
&
rpcMsg
,
&
rpcRsp
);
SRetrieveFuncRsp
retrieveRsp
=
{
0
};
tDeserializeSRetrieveFuncRsp
(
rpcRsp
.
pCont
,
rpcRsp
.
contLen
,
&
retrieveRsp
);
SFuncInfo
*
pFuncInfo
=
(
SFuncInfo
*
)
taosArrayGet
(
retrieveRsp
.
pFuncInfos
,
0
);
taosArrayDestroy
(
retrieveRsp
.
pFuncInfos
);
rpcFreeCont
(
rpcRsp
.
pCont
);
return
0
;
}
int32_t
openUdfdClientRpc
(
SUdfdContext
*
ctx
)
{
char
*
pass
=
"taosdata"
;
char
*
user
=
"root"
;
char
secretEncrypt
[
TSDB_PASSWORD_LEN
+
1
]
=
{
0
};
taosEncryptPass_c
((
uint8_t
*
)
pass
,
strlen
(
pass
),
secretEncrypt
);
SRpcInit
rpcInit
=
{
0
};
rpcInit
.
label
=
(
char
*
)
"UDFD"
;
rpcInit
.
numOfThreads
=
1
;
rpcInit
.
cfp
=
udfdProcessRpcRsp
;
rpcInit
.
sessions
=
1024
;
rpcInit
.
connType
=
TAOS_CONN_CLIENT
;
rpcInit
.
idleTime
=
30
*
1000
;
rpcInit
.
parent
=
ctx
;
rpcInit
.
user
=
(
char
*
)
user
;
rpcInit
.
ckey
=
(
char
*
)
"key"
;
rpcInit
.
secret
=
(
char
*
)
secretEncrypt
;
rpcInit
.
spi
=
1
;
ctx
->
clientRpc
=
rpcOpen
(
&
rpcInit
);
return
0
;
}
int32_t
closeUdfdClientRpc
(
SUdfdContext
*
ctx
)
{
rpcClose
(
ctx
->
clientRpc
);
return
0
;
}
int
main
()
{
debugPrint
(
"libuv version: %x"
,
UV_VERSION_HEX
);
...
...
source/libs/function/test/runUdf.c
浏览文件 @
b62b1786
...
...
@@ -5,6 +5,7 @@
#include "uv.h"
#include "os.h"
#include "tudf.h"
#include "tdatablock.h"
int
main
(
int
argc
,
char
*
argv
[])
{
startUdfService
();
...
...
@@ -18,28 +19,38 @@ int main(int argc, char *argv[]) {
}
fprintf
(
stdout
,
"current working directory:%s
\n
"
,
path
);
strcat
(
path
,
"/libudf1.so"
);
SUdfInfo
udfInfo
=
{.
udfName
=
"udf1"
,
.
path
=
path
};
UdfHandle
handle
;
setupUdf
(
&
udfInfo
,
&
handle
);
SEpSet
epSet
;
setupUdf
(
"udf1"
,
&
epSet
,
&
handle
);
//char state[5000000] = "state";
//char input[5000000] = "input";
int
dataSize
=
500
;
int
callCount
=
2
;
if
(
argc
>
1
)
dataSize
=
atoi
(
argv
[
1
]);
if
(
argc
>
2
)
callCount
=
atoi
(
argv
[
2
]);
char
*
state
=
taosMemoryMalloc
(
dataSize
);
char
*
input
=
taosMemoryMalloc
(
dataSize
);
SUdfDataBlock
blockInput
=
{.
data
=
input
,
.
size
=
dataSize
};
SUdfDataBlock
blockOutput
;
char
*
newState
;
int32_t
newStateSize
;
for
(
int
l
=
0
;
l
<
callCount
;
++
l
)
{
callUdf
(
handle
,
0
,
state
,
dataSize
,
blockInput
,
&
newState
,
&
newStateSize
,
&
blockOutput
);
SSDataBlock
block
=
{
0
};
SSDataBlock
*
pBlock
=
&
block
;
pBlock
->
pDataBlock
=
taosArrayInit
(
1
,
sizeof
(
SColumnInfoData
));
pBlock
->
info
.
numOfCols
=
1
;
pBlock
->
info
.
rows
=
4
;
char
data
[
16
]
=
{
0
};
char
bitmap
[
4
]
=
{
0
};
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
numOfCols
;
++
i
)
{
SColumnInfoData
colInfo
=
{
0
};
colInfo
.
info
.
type
=
TSDB_DATA_TYPE_INT
;
colInfo
.
info
.
bytes
=
sizeof
(
int32_t
);
colInfo
.
info
.
colId
=
1
;
colInfo
.
pData
=
data
;
colInfo
.
nullbitmap
=
bitmap
;
for
(
int32_t
j
=
0
;
j
<
pBlock
->
info
.
rows
;
++
j
)
{
colDataAppendInt32
(
&
colInfo
,
j
,
&
j
);
}
taosArrayPush
(
pBlock
->
pDataBlock
,
&
colInfo
);
}
SSDataBlock
output
=
{
0
};
callUdfScalaProcess
(
handle
,
pBlock
,
&
output
);
SColumnInfoData
*
col
=
taosArrayGet
(
output
.
pDataBlock
,
0
);
for
(
int32_t
i
=
0
;
i
<
output
.
info
.
rows
;
++
i
)
{
fprintf
(
stderr
,
"%d
\t
%d
\n
"
,
i
,
*
(
int32_t
*
)(
col
->
pData
+
i
*
sizeof
(
int32_t
)));
}
taosMemoryFree
(
state
);
taosMemoryFree
(
input
);
teardownUdf
(
handle
);
stopUdfService
();
...
...
source/libs/function/test/udf1.c
浏览文件 @
b62b1786
...
...
@@ -2,20 +2,68 @@
#include <stdlib.h>
#include <stdio.h>
#include "os.h"
#include "tudf.h"
void
udf1
(
int8_t
step
,
char
*
state
,
int32_t
stateSize
,
SUdfDataBlock
input
,
char
**
newState
,
int32_t
*
newStateSize
,
SUdfDataBlock
*
output
)
{
fprintf
(
stdout
,
"%s, step:%d
\n
"
,
"udf function called"
,
step
);
char
*
newStateBuf
=
taosMemoryMalloc
(
stateSize
);
memcpy
(
newStateBuf
,
state
,
stateSize
);
*
newState
=
newStateBuf
;
*
newStateSize
=
stateSize
;
char
*
outputBuf
=
taosMemoryMalloc
(
input
.
size
);
memcpy
(
outputBuf
,
input
.
data
,
input
.
size
);
output
->
data
=
outputBuf
;
output
->
size
=
input
.
size
;
return
;
#undef malloc
#define malloc malloc
#undef free
#define free free
int32_t
udf1_setup
()
{
return
0
;
}
int32_t
udf1_teardown
()
{
return
0
;
}
int32_t
udf1
(
SUdfDataBlock
block
,
SUdfColumn
*
resultCol
)
{
SUdfColumnData
*
resultData
=
&
resultCol
->
colData
;
resultData
->
numOfRows
=
block
.
numOfRows
;
SUdfColumnData
*
srcData
=
&
block
.
udfCols
[
0
]
->
colData
;
resultData
->
varLengthColumn
=
srcData
->
varLengthColumn
;
if
(
resultData
->
varLengthColumn
)
{
resultData
->
varLenCol
.
varOffsetsLen
=
srcData
->
varLenCol
.
varOffsetsLen
;
resultData
->
varLenCol
.
varOffsets
=
malloc
(
resultData
->
varLenCol
.
varOffsetsLen
);
memcpy
(
resultData
->
varLenCol
.
varOffsets
,
srcData
->
varLenCol
.
varOffsets
,
srcData
->
varLenCol
.
varOffsetsLen
);
resultData
->
varLenCol
.
payloadLen
=
srcData
->
varLenCol
.
payloadLen
;
resultData
->
varLenCol
.
payload
=
malloc
(
resultData
->
varLenCol
.
payloadLen
);
memcpy
(
resultData
->
varLenCol
.
payload
,
srcData
->
varLenCol
.
payload
,
srcData
->
varLenCol
.
payloadLen
);
}
else
{
resultData
->
fixLenCol
.
nullBitmapLen
=
srcData
->
fixLenCol
.
nullBitmapLen
;
resultData
->
fixLenCol
.
nullBitmap
=
malloc
(
resultData
->
fixLenCol
.
nullBitmapLen
);
memcpy
(
resultData
->
fixLenCol
.
nullBitmap
,
srcData
->
fixLenCol
.
nullBitmap
,
srcData
->
fixLenCol
.
nullBitmapLen
);
resultData
->
fixLenCol
.
dataLen
=
srcData
->
fixLenCol
.
dataLen
;
resultData
->
fixLenCol
.
data
=
malloc
(
resultData
->
fixLenCol
.
dataLen
);
memcpy
(
resultData
->
fixLenCol
.
data
,
srcData
->
fixLenCol
.
data
,
srcData
->
fixLenCol
.
dataLen
);
for
(
int32_t
i
=
0
;
i
<
resultData
->
numOfRows
;
++
i
)
{
*
(
resultData
->
fixLenCol
.
data
+
i
*
sizeof
(
int32_t
))
=
88
;
}
}
SUdfColumnMeta
*
meta
=
&
resultCol
->
colMeta
;
meta
->
bytes
=
4
;
meta
->
type
=
TSDB_DATA_TYPE_INT
;
meta
->
scale
=
0
;
meta
->
precision
=
0
;
return
0
;
}
int32_t
udf1_free
(
SUdfColumn
*
col
)
{
SUdfColumnData
*
data
=
&
col
->
colData
;
if
(
data
->
varLengthColumn
)
{
free
(
data
->
varLenCol
.
varOffsets
);
data
->
varLenCol
.
varOffsets
=
NULL
;
free
(
data
->
varLenCol
.
payload
);
data
->
varLenCol
.
payload
=
NULL
;
}
else
{
free
(
data
->
fixLenCol
.
nullBitmap
);
data
->
fixLenCol
.
nullBitmap
=
NULL
;
free
(
data
->
fixLenCol
.
data
);
data
->
fixLenCol
.
data
=
NULL
;
}
return
0
;
}
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录