Merge pull request #11572 from taosdata/3.0_udfd

feat(udf): UDF service refactoring with new interface

source/libs/function/CMakeLists.txt

@@ -5,6 +5,9 @@ target_include_directories(
     function
     PUBLIC
         "${TD_SOURCE_DIR}/include/libs/function"
+        "${TD_SOURCE_DIR}/include/util"
+        "${TD_SOURCE_DIR}/include/common"
+        "${TD_SOURCE_DIR}/include/client"
         "${TD_SOURCE_DIR}/contrib/libuv/include"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
@@ -21,6 +24,9 @@ target_include_directories(
         PUBLIC
             "${TD_SOURCE_DIR}/include/libs/function"
             "${TD_SOURCE_DIR}/contrib/libuv/include"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/client"
             "${TD_SOURCE_DIR}/include/os"
         PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
@@ -35,6 +41,9 @@ target_include_directories(
         udf1
         PUBLIC
             "${TD_SOURCE_DIR}/include/libs/function"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/client"
             "${TD_SOURCE_DIR}/include/os"
         PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
@@ -46,6 +55,10 @@ target_include_directories(
         PUBLIC
             "${TD_SOURCE_DIR}/include/libs/function"
             "${TD_SOURCE_DIR}/contrib/libuv/include"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/libs/transport"
+            "${TD_SOURCE_DIR}/include/client"
         PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/function/inc/tudf.h

@@ -16,12 +16,25 @@
 #ifndef TDENGINE_TUDF_H
 #define TDENGINE_TUDF_H
 
+
+#include <stdint.h>
+#include <stdbool.h>
+#include "tmsg.h"
+#include "tcommon.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 //======================================================================================
 //begin API to taosd and qworker
+
+enum {
+  UDFC_CODE_STOPPING = -1,
+  UDFC_CODE_RESTARTING = -2,
+  UDFC_CODE_PIPE_READ_ERR = -3,
+};
+
 /**
  * start udf dameon service
  * @return error code
@@ -34,28 +47,6 @@ int32_t startUdfService();
  */
 int32_t stopUdfService();
 
-enum {
-  TSDB_UDF_TYPE_SCALAR = 0,
-  TSDB_UDF_TYPE_AGGREGATE = 1
-};
-
-enum {
-  TSDB_UDF_SCRIPT_BIN_LIB = 0,
-  TSDB_UDF_SCRIPT_LUA = 1,
-};
-
-typedef struct SUdfInfo {
-  char   *udfName;        // function name
-  int32_t udfType;    // scalar function or aggregate function
-  int8_t    scriptType;
-  char *path;
-
-  int8_t  resType;     // result type
-  int16_t resBytes;    // result byte
-  int32_t bufSize;     //interbuf size
-
-} SUdfInfo;
-
 typedef void *UdfHandle;
 
 /**
@@ -64,36 +55,67 @@ typedef void *UdfHandle;
  * @param handle, out
  * @return error code
  */
-int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle);
-
-
-enum {
-  TSDB_UDF_STEP_NORMAL = 0,
-  TSDB_UDF_STEP_MERGE,
-  TSDb_UDF_STEP_FINALIZE,
-  TSDB_UDF_STEP_MAX_NUM
-};
-/**
- * call udf
- * @param handle udf handle
- * @param step
- * @param state
- * @param stateSize
- * @param input
- * @param newstate
- * @param newStateSize
- * @param output
- * @return error code
- */
+int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle);
+
+typedef struct SUdfColumnMeta {
+  int16_t type;
+  int32_t bytes; // <0 var length, others fixed length bytes
+  uint8_t precision;
+  uint8_t scale;
+} SUdfColumnMeta;
+
+typedef struct SUdfColumnData {
+  int32_t numOfRows;
+  bool varLengthColumn;
+  union {
+    struct {
+      int32_t nullBitmapLen;
+      char   *nullBitmap;
+      int32_t dataLen;
+      char   *data;
+    } fixLenCol;
+
+    struct {
+      int32_t varOffsetsLen;
+      char   *varOffsets;
+      int32_t payloadLen;
+      char   *payload;
+    } varLenCol;
+  };
+} SUdfColumnData;
+
+
+typedef struct SUdfColumn {
+  SUdfColumnMeta colMeta;
+  SUdfColumnData colData;
+} SUdfColumn;
 
-//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined
 typedef struct SUdfDataBlock {
-  char* data;
-  int32_t size;
+  int32_t numOfRows;
+  int32_t numOfCols;
+  SUdfColumn **udfCols;
 } SUdfDataBlock;
 
-int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate,
-                int32_t *newStateSize, SUdfDataBlock *output);
+typedef struct SUdfInterBuf {
+  int32_t bufLen;
+  char* buf;
+} SUdfInterBuf;
+
+//TODO: translate these calls to callUdf
+// output: interBuf
+int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
+// input: block, state
+// output: newState
+int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
+// input: interBuf
+// output: resultData
+int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
+// input: interbuf1, interbuf2
+// output: resultBuf
+int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
+// input: block
+// output: resultData
+int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData);
 
 /**
  * tearn down udf
@@ -104,29 +126,25 @@ int32_t teardownUdf(UdfHandle handle);
 
 // end API to taosd and qworker
 //=============================================================================================================================
-// TODO: Must change
 // begin API to UDF writer.
 
-// script
+// dynamic lib init and destroy
+typedef int32_t (*TUdfSetupFunc)();
+typedef int32_t (*TUdfTeardownFunc)();
 
-//typedef int32_t (*scriptInitFunc)(void* pCtx);
-//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
-//                                 int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
-//                                 int16_t oType, int16_t oBytes);
-//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
-//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
-//typedef void (*scriptDestroyFunc)(void* pCtx);
+//TODO: add API to check function arguments type, number etc.
+//TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory.
+// then UDF framework will free the memory
+//typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data);
+//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
 
-// dynamic lib
-typedef int32_t (*TUdfInitFunc)();
-typedef void (*TUdfDestroyFunc)();
+typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
 
-typedef void (*TUdfFunc)(int8_t step,
-                         char *state, int32_t stateSize, SUdfDataBlock input,
-                         char **newstate, int32_t *newStateSize, SUdfDataBlock *output);
+typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol);
+typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf);
+typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf);
+typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
 
-//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput);
-//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output);
 
 // end API to UDF writer
 //=======================================================================================================================

source/libs/function/inc/tudfInt.h

@@ -15,7 +15,6 @@
 
 #ifndef TDENGINE_TUDF_INT_H
 #define TDENGINE_TUDF_INT_H
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -30,36 +29,37 @@ enum {
 
 };
 
+enum {
+  TSDB_UDF_CALL_AGG_INIT = 0,
+  TSDB_UDF_CALL_AGG_PROC,
+  TSDB_UDF_CALL_AGG_MERGE,
+  TSDB_UDF_CALL_AGG_FIN,
+  TSDB_UDF_CALL_SCALA_PROC,
+};
+
 typedef struct SUdfSetupRequest {
-  char udfName[16]; //
-  int8_t scriptType; // 0:c, 1: lua, 2:js
-  int8_t udfType; //udaf, udf
-  int16_t pathSize;
-  char *path;
+  char udfName[TSDB_FUNC_NAME_LEN];
+  SEpSet epSet;
 } SUdfSetupRequest;
 
 typedef struct SUdfSetupResponse {
   int64_t udfHandle;
 } SUdfSetupResponse;
 
-
 typedef struct SUdfCallRequest {
   int64_t udfHandle;
-  int8_t step;
+  int8_t callType;
 
-  int32_t inputBytes;
-  char *input;
-
-  int32_t stateBytes;
-  char *state;
+  SSDataBlock block;
+  SUdfInterBuf interBuf;
+  SUdfInterBuf interBuf2;
+  int8_t initFirst;
 } SUdfCallRequest;
 
-
 typedef struct SUdfCallResponse {
-  int32_t outputBytes;
-  char *output;
-  int32_t newStateBytes;
-  char *newState;
+  int8_t callType;
+  SSDataBlock resultData;
+  SUdfInterBuf resultBuf;
 } SUdfCallResponse;
 
 
@@ -76,7 +76,11 @@ typedef struct SUdfRequest {
   int64_t seqNum;
 
   int8_t type;
-  void *subReq;
+  union {
+    SUdfSetupRequest setup;
+    SUdfCallRequest call;
+    SUdfTeardownRequest teardown;
+  };
 } SUdfRequest;
 
 typedef struct SUdfResponse {
@@ -85,13 +89,25 @@ typedef struct SUdfResponse {
 
   int8_t type;
   int32_t code;
-  void *subRsp;
+  union {
+    SUdfSetupResponse setupRsp;
+    SUdfCallResponse callRsp;
+    SUdfTeardownResponse teardownRsp;
+  };
 } SUdfResponse;
 
-int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest);
-int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response);
-int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request);
-int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse);
+int32_t encodeUdfRequest(void **buf, const SUdfRequest* request);
+void* decodeUdfRequest(const void *buf, SUdfRequest* request);
+
+int32_t encodeUdfResponse(void **buf, const SUdfResponse *response);
+void* decodeUdfResponse(const void* buf, SUdfResponse *response);
+
+void freeUdfColumnData(SUdfColumnData *data);
+void freeUdfColumn(SUdfColumn* col);
+void freeUdfDataDataBlock(SUdfDataBlock *block);
+
+int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
+int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
 
 #ifdef __cplusplus
 }

source/libs/function/src/udfd.c

@@ -20,6 +20,10 @@
 #include "tudf.h"
 #include "tudfInt.h"
 
+#include "tdataformat.h"
+#include "tglobal.h"
+#include "tmsg.h"
+#include "trpc.h"
 
 static uv_loop_t *loop;
 
@@ -44,7 +48,8 @@ typedef struct SUdf {
     int8_t type;
 
     uv_lib_t lib;
-    TUdfFunc normalFunc;
+    TUdfScalarProcFunc scalarProcFunc;
+    TUdfFreeUdfColumnFunc freeUdfColumn;
 } SUdf;
 
 //TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@@ -56,119 +61,120 @@ typedef struct SUdfHandle {
 
 void udfdProcessRequest(uv_work_t *req) {
     SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data);
-    SUdfRequest *request = NULL;
-    decodeRequest(uvUdf->input.base, uvUdf->input.len, &request);
+    SUdfRequest request = {0};
+    decodeUdfRequest(uvUdf->input.base, &request);
 
-    switch (request->type) {
+    switch (request.type) {
         case UDF_TASK_SETUP: {
             debugPrint("%s", "process setup request");
             SUdf *udf = taosMemoryMalloc(sizeof(SUdf));
             udf->refCount = 0;
-            SUdfSetupRequest *setup = request->subReq;
+            SUdfSetupRequest *setup = &request.setup;
             strcpy(udf->name, setup->udfName);
-            int err = uv_dlopen(setup->path, &udf->lib);
+            //TODO: retrive udf info from mnode
+            char* path = "libudf1.so";
+            int err = uv_dlopen(path, &udf->lib);
             if (err != 0) {
-                debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err));
+                debugPrint("can not load library %s. error: %s", path, uv_strerror(err));
                 //TODO set error
             }
 
-            char normalFuncName[32] = {0};
+            char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
             strcpy(normalFuncName, setup->udfName);
-	    //TODO error,
+	    //TODO error, multi-thread, same udf, lock it
 	    //TODO find all functions normal, init, destroy, normal, merge, finalize
-            uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc));
+            uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->scalarProcFunc));
+            char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
+            char *freeSuffix = "_free";
+            strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
+            strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
+            uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
 
             SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
             handle->udf = udf;
             udf->refCount++;
             //TODO: allocate private structure and call init function and set it to handle
-            SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse));
-            rsp->seqNum = request->seqNum;
-            rsp->type = request->type;
-            rsp->code = 0;
-            SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse));
-            subRsp->udfHandle = (int64_t) (handle);
-            rsp->subRsp = subRsp;
-            char *buf;
-            int32_t len;
-            encodeResponse(&buf, &len, rsp);
-
-            uvUdf->output = uv_buf_init(buf, len);
-
-            taosMemoryFree(rsp->subRsp);
-            taosMemoryFree(rsp);
-            taosMemoryFree(request->subReq);
-            taosMemoryFree(request);
+            SUdfResponse rsp;
+            rsp.seqNum = request.seqNum;
+            rsp.type = request.type;
+            rsp.code = 0;
+            rsp.setupRsp.udfHandle = (int64_t) (handle);
+            int32_t len = encodeUdfResponse(NULL, &rsp);
+            rsp.msgLen = len;
+            void *bufBegin = taosMemoryMalloc(len);
+            void *buf = bufBegin;
+            encodeUdfResponse(&buf, &rsp);
+
+            uvUdf->output = uv_buf_init(bufBegin, len);
+
             taosMemoryFree(uvUdf->input.base);
             break;
         }
 
         case UDF_TASK_CALL: {
             debugPrint("%s", "process call request");
-            SUdfCallRequest *call = request->subReq;
+            SUdfCallRequest *call = &request.call;
             SUdfHandle *handle = (SUdfHandle *) (call->udfHandle);
             SUdf *udf = handle->udf;
-            char *newState;
-            int32_t newStateSize;
-            SUdfDataBlock input = {.data = call->input, .size= call->inputBytes};
-            SUdfDataBlock output;
-	    //TODO: call different functions according to the step 
-            udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output);
-
-            SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse));
-            rsp->seqNum = request->seqNum;
-            rsp->type = request->type;
-            rsp->code = 0;
-            SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse));
-            subRsp->outputBytes = output.size;
-            subRsp->output = output.data;
-            subRsp->newStateBytes = newStateSize;
-            subRsp->newState = newState;
-            rsp->subRsp = subRsp;
-
-            char *buf;
-            int32_t len;
-            encodeResponse(&buf, &len, rsp);
-            uvUdf->output = uv_buf_init(buf, len);
-
-            taosMemoryFree(rsp->subRsp);
-            taosMemoryFree(rsp);
-            taosMemoryFree(newState);
-            taosMemoryFree(output.data);
-            taosMemoryFree(request->subReq);
-            taosMemoryFree(request);
+
+            SUdfDataBlock input = {0};
+            convertDataBlockToUdfDataBlock(&call->block, &input);
+            SUdfColumn output = {0};
+	    //TODO: call different functions according to call type, for now just calar
+            if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
+              udf->scalarProcFunc(input, &output);
+            }
+
+            SUdfResponse  response = {0};
+            SUdfResponse *rsp = &response;
+            if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
+              rsp->seqNum = request.seqNum;
+              rsp->type = request.type;
+              rsp->code = 0;
+              SUdfCallResponse *subRsp = &rsp->callRsp;
+              subRsp->callType = call->callType;
+              convertUdfColumnToDataBlock(&output, &subRsp->resultData);
+            }
+
+            int32_t len = encodeUdfResponse(NULL, rsp);
+            rsp->msgLen = len;
+            void *bufBegin = taosMemoryMalloc(len);
+            void *buf = bufBegin;
+            encodeUdfResponse(&buf, rsp);
+            uvUdf->output = uv_buf_init(bufBegin, len);
+
+            //TODO: free
+            udf->freeUdfColumn(&output);
+
             taosMemoryFree(uvUdf->input.base);
             break;
         }
         case UDF_TASK_TEARDOWN: {
             debugPrint("%s", "process teardown request");
 
-            SUdfTeardownRequest *teardown = request->subReq;
+            SUdfTeardownRequest *teardown = &request.teardown;
             SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle);
             SUdf *udf = handle->udf;
             udf->refCount--;
             if (udf->refCount == 0) {
                 uv_dlclose(&udf->lib);
+                taosMemoryFree(udf);
             }
-            taosMemoryFree(udf);
-	    //TODO: call destroy and free udf private 
+	    //TODO: call destroy and free udf private
             taosMemoryFree(handle);
 
-            SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse));
-            rsp->seqNum = request->seqNum;
-            rsp->type = request->type;
+            SUdfResponse response;
+            SUdfResponse *rsp = &response;
+            rsp->seqNum = request.seqNum;
+            rsp->type = request.type;
             rsp->code = 0;
-            SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse));
-            rsp->subRsp = subRsp;
-            char *buf;
-            int32_t len;
-            encodeResponse(&buf, &len, rsp);
-            uvUdf->output = uv_buf_init(buf, len);
-
-            taosMemoryFree(rsp->subRsp);
-            taosMemoryFree(rsp);
-            taosMemoryFree(request->subReq);
-            taosMemoryFree(request);
+            int32_t len = encodeUdfResponse(NULL, rsp);
+            rsp->msgLen = len;
+            void *bufBegin = taosMemoryMalloc(len);
+            void *buf = bufBegin;
+            encodeUdfResponse(&buf, rsp);
+            uvUdf->output = uv_buf_init(bufBegin, len);
+
             taosMemoryFree(uvUdf->input.base);
             break;
         }
@@ -181,7 +187,7 @@ void udfdProcessRequest(uv_work_t *req) {
 }
 
 void udfdOnWrite(uv_write_t *req, int status) {
-    debugPrint("%s", "after writing to pipe");
+    debugPrint("%s", "server after writing to pipe");
     if (status < 0) {
         debugPrint("Write error %s", uv_err_name(status));
     }
@@ -205,7 +211,7 @@ void udfdSendResponse(uv_work_t *work, int status) {
 }
 
 void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
-    debugPrint("%s", "allocate buffer for read");
+    debugPrint("%s", "server allocate buffer for read");
     SUdfdUvConn *ctx = handle->data;
     int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
     if (ctx->inputCap == 0) {
@@ -329,6 +335,77 @@ void removeListeningPipe(int sig) {
     exit(0);
 }
 
+typedef struct SServerContext {
+  void *clientRpc;
+} SUdfdContext;
+
+
+void udfdProcessRpcRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet)  {
+
+  return;
+}
+
+int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int32_t numOfUdfs) {
+  SRetrieveFuncReq retrieveReq = {0};
+  retrieveReq.numOfFuncs = 1;
+  retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
+  for (int32_t i = 0; i < numOfUdfs; ++i) {
+    taosArrayPush(retrieveReq.pFuncNames, udfNames[i]);
+  }
+
+  int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
+  taosArrayDestroy(retrieveReq.pFuncNames);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = contLen;
+  rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
+
+  SRpcMsg rpcRsp = {0};
+  rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp);
+  SRetrieveFuncRsp retrieveRsp = {0};
+  tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
+
+  SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
+
+  taosArrayDestroy(retrieveRsp.pFuncInfos);
+
+  rpcFreeCont(rpcRsp.pCont);
+  return 0;
+}
+
+int32_t openUdfdClientRpc(SUdfdContext *ctx) {
+  char *pass = "taosdata";
+  char *user = "root";
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
+  SRpcInit rpcInit = {0};
+  rpcInit.label = (char*)"UDFD";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp = udfdProcessRpcRsp;
+  rpcInit.sessions = 1024;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.idleTime = 30 * 1000;
+  rpcInit.parent = ctx;
+
+  rpcInit.user = (char*)user;
+  rpcInit.ckey = (char*)"key";
+  rpcInit.secret = (char*)secretEncrypt;
+  rpcInit.spi = 1;
+
+  ctx->clientRpc = rpcOpen(&rpcInit);
+
+  return 0;
+}
+
+int32_t closeUdfdClientRpc(SUdfdContext *ctx) {
+  rpcClose(ctx->clientRpc);
+
+  return 0;
+}
+
 int main() {
     debugPrint("libuv version: %x", UV_VERSION_HEX);
 

source/libs/function/test/runUdf.c

@@ -5,6 +5,7 @@
 #include "uv.h"
 #include "os.h"
 #include "tudf.h"
+#include "tdatablock.h"
 
 int main(int argc, char *argv[]) {
     startUdfService();
@@ -18,28 +19,38 @@ int main(int argc, char *argv[]) {
     }
     fprintf(stdout, "current working directory:%s\n", path);
     strcat(path, "/libudf1.so");
-    SUdfInfo udfInfo = {.udfName="udf1", .path=path};
 
     UdfHandle handle;
-    setupUdf(&udfInfo, &handle);
+    SEpSet epSet;
+    setupUdf("udf1", &epSet, &handle);
 
-    //char state[5000000] = "state";
-    //char input[5000000] = "input";
-    int dataSize = 500;
-    int callCount = 2;
-    if (argc > 1) dataSize = atoi(argv[1]);
-    if (argc > 2) callCount = atoi(argv[2]);
-    char *state = taosMemoryMalloc(dataSize);
-    char *input = taosMemoryMalloc(dataSize);
-    SUdfDataBlock blockInput = {.data = input, .size = dataSize};
-    SUdfDataBlock blockOutput;
-    char* newState;
-    int32_t newStateSize;
-    for (int l = 0; l < callCount; ++l) {
-        callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput);
+    SSDataBlock block = {0};
+    SSDataBlock* pBlock = &block;
+    pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+    pBlock->info.numOfCols = 1;
+    pBlock->info.rows = 4;
+    char data[16] = {0};
+    char bitmap[4] = {0};
+    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+      SColumnInfoData colInfo = {0};
+      colInfo.info.type = TSDB_DATA_TYPE_INT;
+      colInfo.info.bytes = sizeof(int32_t);
+      colInfo.info.colId = 1;
+      colInfo.pData = data;
+      colInfo.nullbitmap = bitmap;
+      for (int32_t j = 0; j < pBlock->info.rows; ++j) {
+        colDataAppendInt32(&colInfo, j, &j);
+      }
+      taosArrayPush(pBlock->pDataBlock, &colInfo);
+    }
+    
+    SSDataBlock output = {0};
+    callUdfScalaProcess(handle, pBlock, &output);
+
+    SColumnInfoData *col = taosArrayGet(output.pDataBlock, 0);
+    for (int32_t i = 0; i < output.info.rows; ++i) {
+      fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t)));
     }
-    taosMemoryFree(state);
-    taosMemoryFree(input);
     teardownUdf(handle);
 
     stopUdfService();

source/libs/function/test/udf1.c

@@ -2,20 +2,68 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-#include "os.h"
 #include "tudf.h"
 
-void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input,
-          char **newState, int32_t *newStateSize, SUdfDataBlock *output) {
-    fprintf(stdout, "%s, step:%d\n", "udf function called", step);
-    char *newStateBuf = taosMemoryMalloc(stateSize);
-    memcpy(newStateBuf, state, stateSize);
-    *newState = newStateBuf;
-    *newStateSize = stateSize;
-
-    char *outputBuf = taosMemoryMalloc(input.size);
-    memcpy(outputBuf, input.data, input.size);
-    output->data = outputBuf;
-    output->size = input.size;
-    return;
+#undef malloc
+#define malloc malloc
+#undef free
+#define free free
+
+int32_t udf1_setup() {
+  return 0;
+}
+
+int32_t udf1_teardown() {
+  return 0;
+}
+
+int32_t udf1(SUdfDataBlock block, SUdfColumn *resultCol) {
+  SUdfColumnData *resultData = &resultCol->colData;
+  resultData->numOfRows = block.numOfRows;
+  SUdfColumnData *srcData = &block.udfCols[0]->colData;
+  resultData->varLengthColumn = srcData->varLengthColumn;
+
+  if (resultData->varLengthColumn) {
+    resultData->varLenCol.varOffsetsLen = srcData->varLenCol.varOffsetsLen;
+    resultData->varLenCol.varOffsets = malloc(resultData->varLenCol.varOffsetsLen);
+    memcpy(resultData->varLenCol.varOffsets, srcData->varLenCol.varOffsets, srcData->varLenCol.varOffsetsLen);
+
+    resultData->varLenCol.payloadLen = srcData->varLenCol.payloadLen;
+    resultData->varLenCol.payload = malloc(resultData->varLenCol.payloadLen);
+    memcpy(resultData->varLenCol.payload, srcData->varLenCol.payload, srcData->varLenCol.payloadLen);
+  } else {
+    resultData->fixLenCol.nullBitmapLen = srcData->fixLenCol.nullBitmapLen;
+    resultData->fixLenCol.nullBitmap = malloc(resultData->fixLenCol.nullBitmapLen);
+    memcpy(resultData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmapLen);
+
+    resultData->fixLenCol.dataLen = srcData->fixLenCol.dataLen;
+    resultData->fixLenCol.data = malloc(resultData->fixLenCol.dataLen);
+    memcpy(resultData->fixLenCol.data, srcData->fixLenCol.data, srcData->fixLenCol.dataLen);
+    for (int32_t i = 0; i < resultData->numOfRows; ++i) {
+      *(resultData->fixLenCol.data + i * sizeof(int32_t)) = 88;
+    }
+  }
+
+  SUdfColumnMeta *meta = &resultCol->colMeta;
+  meta->bytes = 4;
+  meta->type = TSDB_DATA_TYPE_INT;
+  meta->scale = 0;
+  meta->precision = 0;
+  return 0;
 }
+
+int32_t udf1_free(SUdfColumn *col) {
+  SUdfColumnData *data = &col->colData;
+  if (data->varLengthColumn) {
+    free(data->varLenCol.varOffsets);
+    data->varLenCol.varOffsets = NULL;
+    free(data->varLenCol.payload);
+    data->varLenCol.payload = NULL;
+  } else {
+    free(data->fixLenCol.nullBitmap);
+    data->fixLenCol.nullBitmap = NULL;
+    free(data->fixLenCol.data);
+    data->fixLenCol.data = NULL;
+  }
+  return 0;
+}
\ No newline at end of file