Merge pull request #6771 from taosdata/feature/cenc

Feature/cenc

cmake/define.inc

@@ -83,6 +83,8 @@ IF (TD_ARM_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm64 is defined")
   SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
 ENDIF ()
 
 IF (TD_ARM_32)
@@ -91,6 +93,8 @@ IF (TD_ARM_32)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm32 is defined")
   SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
+
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
 ENDIF ()
 
 IF (TD_MIPS_64)
@@ -143,6 +147,7 @@ IF (TD_LINUX)
 
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
 ENDIF ()
 
 IF (TD_DARWIN_64)
@@ -164,6 +169,7 @@ IF (TD_DARWIN_64)
   SET(RELEASE_FLAGS "-Og")
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
 ENDIF ()
 
 IF (TD_WINDOWS)
@@ -194,6 +200,7 @@ IF (TD_WINDOWS)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/wepoll/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
 ENDIF ()
 
 IF (TD_WINDOWS_64)

src/client/CMakeLists.txt

@@ -21,7 +21,7 @@ IF (TD_LINUX)
   IF (TD_LINUX_64)
     TARGET_LINK_LIBRARIES(taos lua)
   ENDIF ()
-  
+
   SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
   #set version of .so
@@ -37,13 +37,13 @@ ELSEIF (TD_DARWIN)
 
   # set the static lib name
   ADD_LIBRARY(taos_static STATIC ${SRC})
-  TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m)
+  TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua)
   SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
   SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
   # generate dynamic library (*.dylib)
   ADD_LIBRARY(taos SHARED ${SRC})
-  TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m)
+  TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua)
   SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
   #set version of .dylib
@@ -68,19 +68,19 @@ ELSEIF (TD_WINDOWS)
   IF (NOT TD_GODLL)
     SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def)
   ENDIF ()
-  TARGET_LINK_LIBRARIES(taos trpc tutil query)
+  TARGET_LINK_LIBRARIES(taos trpc tutil query lua)
 
 ELSEIF (TD_DARWIN)
   SET(CMAKE_MACOSX_RPATH 1)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
 
   ADD_LIBRARY(taos_static STATIC ${SRC})
-  TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m)
+  TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua)
   SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
 
   # generate dynamic library (*.dylib)
   ADD_LIBRARY(taos SHARED ${SRC})
-  TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m)
+  TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua)
 
   SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 

src/client/inc/tscUtil.h

@@ -107,6 +107,7 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
                                    uint32_t offset);
 
 void*   tscDestroyBlockArrayList(SArray* pDataBlockList);
+void*  tscDestroyUdfArrayList(SArray* pUdfList);
 void*   tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
 
 int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
@@ -261,6 +262,7 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
 int  tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo);
 int  tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
 int  tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
+int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo);
 
 void tscResetForNextRetrieve(SSqlRes* pRes);
 void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
@@ -310,7 +312,7 @@ void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
 void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
 void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
 int  tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
-int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp);
+int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp);
 
 int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray);
 

src/client/inc/tsclient.h

@@ -146,6 +146,7 @@ typedef struct {
   SInsertStatementParam insertParam;
   char    reserve1[3];        // fix bus error on arm32
   int32_t count;   // todo remove it
+  bool    subCmd;
 
   char         reserve2[3];        // fix bus error on arm32
   int16_t      numOfCols;

src/client/src/tscGlobalmerge.c

@@ -602,6 +602,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
           if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
             continue;
           }
+
+          if (functionId < 0) {
+            SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
+
+            doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
+
+            continue;
+          }
+
           aAggs[functionId].mergeFunc(&pCtx[j]);
         }
       } else {
@@ -610,6 +619,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
           if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
             continue;
           }
+
+          if (functionId < 0) {
+            SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
+
+            doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
+
+            continue;
+          }
+
           aAggs[functionId].xFinalize(&pCtx[j]);
         }
 
@@ -626,7 +644,11 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
         }
 
         for(int32_t j = 0; j < numOfExpr; ++j) {
-          aAggs[pCtx[j].functionId].init(&pCtx[j]);
+          if (pCtx[j].functionId < 0) {
+            continue;
+          }
+
+          aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo);
         }
 
         for (int32_t j = 0; j < numOfExpr; ++j) {
@@ -638,6 +660,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
           if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
             continue;
           }
+
+          if (functionId < 0) {
+            SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
+
+            doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
+
+            continue;
+          }
+
           aAggs[functionId].mergeFunc(&pCtx[j]);
         }
       }
@@ -651,6 +682,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
         if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
           continue;
         }
+
+        if (functionId < 0) {
+          SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
+
+          doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
+
+          continue;
+        }
+
         aAggs[functionId].mergeFunc(&pCtx[j]);
       }
     }
@@ -871,7 +911,6 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
 
   {
     if (pAggInfo->hasDataBlockForNewGroup) {
-      pAggInfo->binfo.pRes->info.rows = 0;
       pAggInfo->hasPrev = false; // now we start from a new group data set.
 
       // not belongs to the same group, return the result of current group;
@@ -880,7 +919,13 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
 
       { // reset output buffer
         for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
-          aAggs[pAggInfo->binfo.pCtx[j].functionId].init(&pAggInfo->binfo.pCtx[j]);
+          SQLFunctionCtx* pCtx = &pAggInfo->binfo.pCtx[j];
+          if (pCtx->functionId < 0) {
+            clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity);
+            continue;
+          }
+
+          aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo);
         }
       }
 
@@ -933,6 +978,14 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
         continue;
       }
 
+      if (functionId < 0) {
+        SUdfInfo* pUdfInfo = taosArrayGet(pAggInfo->udfInfo, -1 * functionId - 1);
+
+        doInvokeUdf(pUdfInfo, &pAggInfo->binfo.pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
+
+        continue;
+      }
+
       aAggs[functionId].xFinalize(&pAggInfo->binfo.pCtx[j]);
     }
 

src/client/src/tscServer.c

@@ -503,6 +503,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) {
       pCmd->command == TSDB_SQL_INSERT ||
       pCmd->command == TSDB_SQL_CONNECT ||
       pCmd->command == TSDB_SQL_HB ||
+      pCmd->command == TSDB_SQL_RETRIEVE_FUNC ||
       pCmd->command == TSDB_SQL_STABLEVGROUP) {
     pRes->code = tscBuildMsg[pCmd->command](pSql, NULL);
   }
@@ -545,7 +546,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
     type = pQueryInfo->type;
 
     // while numOfTables equals to 0, it must be Heartbeat
-    assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0);
+    assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0);
   }
 
   tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type);
@@ -1032,6 +1033,34 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pMsg += sizeof(int32_t);
   }
 
+  // support only one udf
+  if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) {
+    pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload));
+    for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i);
+      *(int8_t*) pMsg = pUdfInfo->resType;
+      pMsg += sizeof(pUdfInfo->resType);
+
+      *(int16_t*) pMsg = htons(pUdfInfo->resBytes);
+      pMsg += sizeof(pUdfInfo->resBytes);
+
+      STR_TO_VARSTR(pMsg, pUdfInfo->name);
+
+      pMsg += varDataTLen(pMsg);
+
+      *(int32_t*) pMsg = htonl(pUdfInfo->funcType);
+      pMsg += sizeof(pUdfInfo->funcType);
+
+      *(int32_t*) pMsg = htonl(pUdfInfo->bufSize);
+      pMsg += sizeof(pUdfInfo->bufSize);
+
+      pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen);
+      memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen);
+
+      pMsg += pUdfInfo->contLen;
+    }
+  }
+
   memcpy(pMsg, pSql->sqlstr, sqlLen);
   pMsg += sqlLen;
 
@@ -1067,6 +1096,18 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+  SSqlCmd *pCmd = &pSql->cmd;
+  SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload;
+
+  pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_FUNCTION;
+
+  pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCreateDnodeMsg);
@@ -1191,6 +1232,17 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_FUNCTION;
+
+  pCmd->payloadLen = sizeof(SDropFuncMsg);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->payloadLen = sizeof(SCMDropTableMsg);
@@ -1294,11 +1346,19 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
+  SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
+  SShowMsg  *pShowMsg = (SShowMsg *)pCmd->payload;
 
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd,  0);
+  if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) {
+    pShowMsg->type = pShowInfo->showType;
+    pShowMsg->payloadLen = 0;
+    pCmd->payloadLen = sizeof(SShowMsg);
 
-  if (tNameIsEmpty(&pTableMetaInfo->name)) {    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (tNameIsEmpty(&pTableMetaInfo->name)) {
     pthread_mutex_lock(&pObj->mutex);
     tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));  
     pthread_mutex_unlock(&pObj->mutex);
@@ -1306,7 +1366,6 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db);
   }
 
-  SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
   pShowMsg->type = pShowInfo->showType;
 
   if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
@@ -1819,6 +1878,29 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  char *pMsg = pCmd->payload;
+  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
+  int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo);
+
+  SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg;
+  pRetrieveFuncMsg->num = htonl(numOfFuncs);
+
+  pMsg += sizeof(SRetrieveFuncMsg);
+  for(int32_t i = 0; i < numOfFuncs; ++i) {
+    SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i);
+    STR_TO_NET_VARSTR(pMsg, pUdf->name);
+    pMsg += varDataNetTLen(pMsg);
+  }
+
+  pCmd->msgType = TSDB_MSG_TYPE_CM_RETRIEVE_FUNC;
+  pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   SSqlCmd *pCmd = &pSql->cmd;
   STscObj *pObj = pSql->pTscObj;
@@ -2034,12 +2116,64 @@ static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t
   return pVgroupInfo;
 }
 
+int tscProcessRetrieveFuncRsp(SSqlObj* pSql) {
+  SSqlCmd* pCmd = &pSql->cmd;
+  SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp;
+  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
+
+  pFuncMsg->num = htonl(pFuncMsg->num);
+  assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo));
+
+  char* pMsg = pFuncMsg->content;
+  for(int32_t i = 0; i < pFuncMsg->num; ++i) {
+    SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
+
+    for(int32_t j = 0; j < pFuncMsg->num; ++j) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
+      if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
+        continue;
+      }
+
+      if (pUdfInfo->content) {
+        continue;
+      }
+
+      pUdfInfo->resBytes = htons(pFunc->resBytes);
+      pUdfInfo->resType  = pFunc->resType;
+      pUdfInfo->funcType = htonl(pFunc->funcType);
+      pUdfInfo->contLen  = htonl(pFunc->len);
+      pUdfInfo->bufSize  = htonl(pFunc->bufSize);
+
+      pUdfInfo->content = malloc(pUdfInfo->contLen);
+      memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
+
+      pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
+    }
+  }
+
+  // master sqlObj locates in param
+  SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param);
+  if(parent == NULL) {
+    return pSql->res.code;
+  }
+
+  SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd);
+
+  assert(parent->signature == parent && (int64_t)pSql->param == parent->self);
+  taosArrayDestroy(parQueryInfo->pUdfInfo);
+
+  parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo;   // assigned to parent sql obj.
+  pQueryInfo->pUdfInfo = NULL;
+  return TSDB_CODE_SUCCESS;
+}
+
 int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
   char *rsp = pSql->res.pRsp;
 
   SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
   pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
   pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup);
+  pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf);
 
   rsp += sizeof(SMultiTableMeta);
 
@@ -2128,6 +2262,37 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
     pMsg += size;
   }
 
+  SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd);
+  if (pMultiMeta->numOfUdf > 0) {
+    assert(pQueryInfo->pUdfInfo != NULL);
+  }
+
+  for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) {
+    SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
+
+    for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
+      if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
+        continue;
+      }
+
+      if (pUdfInfo->content) {
+        continue;
+      }
+
+      pUdfInfo->resBytes = htons(pFunc->resBytes);
+      pUdfInfo->resType  = pFunc->resType;
+      pUdfInfo->funcType = htonl(pFunc->funcType);
+      pUdfInfo->contLen  = htonl(pFunc->len);
+      pUdfInfo->bufSize  = htonl(pFunc->bufSize);
+
+      pUdfInfo->content = malloc(pUdfInfo->contLen);
+      memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
+
+      pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
+    }
+  }
+
   pSql->res.code = TSDB_CODE_SUCCESS;
   pSql->res.numOfTotal = pMultiMeta->numOfTables;
   tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables);
@@ -2518,7 +2683,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn
   return code;
 }
 
-int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp) {
+int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp) {
   SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
   if (NULL == pNew) {
     tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self);
@@ -2531,8 +2696,9 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
 
   int32_t numOfTable      = (int32_t) taosArrayGetSize(pNameList);
   int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList);
+  int32_t numOfUdf        = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0;
 
-  int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + sizeof(SMultiTableInfoMsg);
+  int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg);
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) {
     tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self);
     tscFreeSqlObj(pNew);
@@ -2542,12 +2708,13 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
   SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload;
   pInfo->numOfTables  = htonl((uint32_t) taosArrayGetSize(pNameList));
   pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList));
+  pInfo->numOfUdfs    = htonl(numOfUdf);
 
   char* start = pInfo->tableNames;
   int32_t len = 0;
   for(int32_t i = 0; i < numOfTable; ++i) {
     char* name = taosArrayGetP(pNameList, i);
-    if (i < numOfTable - 1 || numOfVgroupList > 0) {
+    if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) {
       len = sprintf(start, "%s,", name);
     } else {
       len = sprintf(start, "%s", name);
@@ -2558,7 +2725,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
 
   for(int32_t i = 0; i < numOfVgroupList; ++i) {
     char* name = taosArrayGetP(pVgroupNameList, i);
-    if (i < numOfVgroupList - 1) {
+    if (i < numOfVgroupList - 1 || numOfUdf > 0) {
       len = sprintf(start, "%s,", name);
     } else {
       len = sprintf(start, "%s", name);
@@ -2567,12 +2734,23 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
     start += len;
   }
 
+  for(int32_t i = 0; i < numOfUdf; ++i) {
+    SUdfInfo * u = taosArrayGet(pUdfList, i);
+    if (i < numOfUdf - 1) {
+      len = sprintf(start, "%s,", u->name);
+    } else {
+      len = sprintf(start, "%s", u->name);
+    }
+
+    start += len;
+  }
+
   pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg));
   pNew->cmd.msgType = TSDB_MSG_TYPE_CM_TABLES_META;
 
   registerSqlObj(pNew);
-  tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, msg size:%d", pSql->self,
-      pNew->self, numOfTable, numOfVgroupList, pNew->cmd.payloadLen);
+  tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self,
+      pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen);
 
   pNew->fp = fp;
   pNew->param = (void *)pSql->self;
@@ -2643,6 +2821,60 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create
   return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists);
 }
 
+int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
+  SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
+  if (NULL == pNew) {
+    tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  pNew->pTscObj = pSql->pTscObj;
+  pNew->signature = pNew;
+  pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC;
+
+  if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) {
+    tscError("%p malloc failed for new queryinfo", pSql);
+    tscFreeSqlObj(pNew);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd);
+
+  pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo));
+  for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
+    SUdfInfo info = {0};
+    SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i);
+    info = *p1;
+    info.name = strdup(p1->name);
+    taosArrayPush(pNewQueryInfo->pUdfInfo, &info);
+  }
+
+  pNew->cmd.active = pNewQueryInfo;
+
+  if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
+    tscError("%p malloc failed for payload to get table meta", pSql);
+    tscFreeSqlObj(pNew);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew);
+  registerSqlObj(pNew);
+
+  pNew->fp = tscTableMetaCallBack;
+  pNew->param = (void *)pSql->self;
+
+  tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self);
+
+  pSql->metaRid = pNew->self;
+
+  int32_t code = tscBuildAndSendRequest(pNew, NULL);
+  if (code == TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify application that current process needs to be terminated
+  }
+
+  return code;
+}
+
 /**
  * retrieve table meta from mnode, and then update the local table meta hashmap.
  * @param pSql          sql object
@@ -2744,6 +2976,7 @@ void tscInitMsgsFp() {
 
   tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg;
   tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg;
+  tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg;
 
   tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg;
   tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg;
@@ -2752,6 +2985,7 @@ void tscInitMsgsFp() {
   tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg;
   tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg;
   tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg;
+  tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg;
   tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg;
   tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg;
   tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg;
@@ -2767,6 +3001,7 @@ void tscInitMsgsFp() {
   tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
 //  tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
   tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg;
+  tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg;
 
   tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
   tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg;
@@ -2785,6 +3020,7 @@ void tscInitMsgsFp() {
   tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp;
   tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp;
   tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp;
+  tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp;
 
   tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
   tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode;  // rsp handled by same function.
@@ -2814,3 +3050,4 @@ void tscInitMsgsFp() {
   tscKeepConn[TSDB_SQL_FETCH] = 1;
   tscKeepConn[TSDB_SQL_HB] = 1;
 }
+

src/client/src/tscSql.c

@@ -566,7 +566,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) {
 
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
 
-  if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+  if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) {
     return true;
   }
 
@@ -679,7 +679,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
 
   SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
 
-  if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+  if (!pQueryInfo->globalMerge) {
     return;
   }
 
@@ -730,7 +730,7 @@ void taos_stop_query(TAOS_RES *res) {
 
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
 
-  if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+  if (pQueryInfo->globalMerge) {
     assert(pSql->rpcRid <= 0);
     tscKillSTableQuery(pSql);
   } else {
@@ -979,7 +979,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   registerSqlObj(pSql);
   tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj);
 
-  code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, loadMultiTableMetaCallback);
+  code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback);
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
     code = TSDB_CODE_SUCCESS;
   }

src/client/src/tscSubquery.c

@@ -23,6 +23,7 @@
 #include "tscSubquery.h"
 #include "qTableMeta.h"
 #include "tsclient.h"
+#include "qUdf.h"
 #include "qUtil.h"
 #include "qPlan.h"
 
@@ -32,7 +33,7 @@ typedef struct SInsertSupporter {
 } SInsertSupporter;
 
 static void freeJoinSubqueryObj(SSqlObj* pSql);
-static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
+//static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
 
 static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
   if (left == right) {
@@ -1896,7 +1897,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
       int16_t type  = 0;
       int32_t inter = 0;
 
-      getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
+      getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL);
 
       SSchema s1 = {.colId = s->colId, .type = (uint8_t)type, .bytes = bytes};
       pSupporter->tagSize = s1.bytes;
@@ -2801,6 +2802,28 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
 
   tscFreeRetrieveSup(pSql);
 
+  // set the command flag must be after the semaphore been correctly set.
+  if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
+    pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE;
+
+    SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd);
+
+    size_t size = tscNumOfExprs(pQueryInfo);
+    for (int32_t j = 0; j < size; ++j) {
+      SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j);
+
+      int32_t functionId = pExprInfo->base.functionId;
+      if (functionId < 0) {
+        SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1);
+        code = initUdfInfo(pUdfInfo);
+        if (code != TSDB_CODE_SUCCESS) {
+          pParentSql->res.code = code;
+          tscAsyncResultOnError(pParentSql);
+        }
+      }
+    }
+  }
+
   if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
     (*pParentSql->fp)(pParentSql->param, pParentSql, 0);
   } else {
@@ -3469,20 +3492,20 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
   if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
     bool allSubqueryExhausted = true;
-    
+
     for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
       if (pSql->pSubs[i] == NULL) {
         continue;
       }
-      
+
       SSqlRes *pRes1 = &pSql->pSubs[i]->res;
       SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd;
-      
+
       SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1);
       assert(pQueryInfo1->numOfTables == 1);
-      
+
       STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0);
-      
+
       /*
        * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are
        * available, goes on
@@ -3493,14 +3516,14 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
         break;
       }
     }
-    
+
     hasData = !allSubqueryExhausted;
   } else {  // otherwise, in case inner join, if any subquery exhausted, query completed.
     for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
       if (pSql->pSubs[i] == 0) {
         continue;
       }
-      
+
       SSqlRes *   pRes1 = &pSql->pSubs[i]->res;
       SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd);
       

src/client/src/tscSystem.c

@@ -28,6 +28,7 @@
 #include "tconfig.h"
 #include "ttimezone.h"
 #include "tlocale.h"
+#include "qScript.h"
 
 // global, not configurable
 #define TSC_VAR_NOT_RELEASE 1
@@ -148,6 +149,8 @@ void taos_init_imp(void) {
     taosInitNotes();
 
     rpcInit();
+
+    scriptEnvPoolInit();
     tscDebug("starting to initialize TAOS client ...");
     tscDebug("Local End Point is:%s", tsLocalEp);
   }
@@ -202,7 +205,9 @@ void taos_cleanup(void) {
   if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
     return;
   }
-
+  if (tscEmbedded == 0) {
+    scriptEnvPoolCleanup();
+  }
   taosHashCleanup(tscTableMetaInfo);
   tscTableMetaInfo = NULL;
 

src/client/src/tscUtil.c

@@ -34,54 +34,54 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
 
 int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
   int32_t n = 0;
-  
+
   switch (type) {
     case TSDB_DATA_TYPE_NULL:
       n = sprintf(str, "null");
       break;
-  
+
     case TSDB_DATA_TYPE_BOOL:
       n = sprintf(str, (*(int8_t*)buf) ? "true" : "false");
       break;
-  
+
     case TSDB_DATA_TYPE_TINYINT:
       n = sprintf(str, "%d", *(int8_t*)buf);
       break;
-  
+
     case TSDB_DATA_TYPE_SMALLINT:
       n = sprintf(str, "%d", *(int16_t*)buf);
       break;
-  
+
     case TSDB_DATA_TYPE_INT:
       n = sprintf(str, "%d", *(int32_t*)buf);
       break;
-  
+
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP:
       n = sprintf(str, "%" PRId64, *(int64_t*)buf);
       break;
-  
+
     case TSDB_DATA_TYPE_FLOAT:
       n = sprintf(str, "%f", GET_FLOAT_VAL(buf));
       break;
-  
+
     case TSDB_DATA_TYPE_DOUBLE:
       n = sprintf(str, "%f", GET_DOUBLE_VAL(buf));
       break;
-  
+
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
       if (bufSize < 0) {
         tscError("invalid buf size");
         return TSDB_CODE_TSC_INVALID_VALUE;
       }
-      
+
       *str = '"';
       memcpy(str + 1, buf, bufSize);
       *(str + bufSize + 1) = '"';
       n = bufSize + 2;
       break;
-  
+
     default:
       tscError("unsupported type:%d", type);
       return TSDB_CODE_TSC_INVALID_VALUE;
@@ -216,6 +216,15 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   for (int32_t i = 0; i < numOfExprs; ++i) {
     int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId;
 
+    if (functionId < 0) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
+      if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
+        return false;
+      }
+
+      continue;
+    }
+
     if (functionId != TSDB_FUNC_PRJ &&
         functionId != TSDB_FUNC_TAGPRJ &&
         functionId != TSDB_FUNC_TAG &&
@@ -266,6 +275,16 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
         f != TSDB_FUNC_DERIVATIVE) {
       return false;
     }
+
+    if (f < 0) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1);
+      if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
+        return false;
+      }
+
+      continue;
+    }
+
   }
 
   return true;
@@ -338,6 +357,10 @@ bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) {
     }
   }
 
+  if (tscIsProjectionQuery(pQueryInfo)) {
+    return false;
+  }
+
   return false;
 }
 
@@ -526,6 +549,15 @@ bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) {
     }
 
     int32_t functionId = pExpr->base.functionId;
+    if (functionId < 0) {
+      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
+      if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
+        return true;
+      }
+
+      continue;
+    }
+
     if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) {
       continue;
     }
@@ -1186,7 +1218,7 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
     SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
 
     pOutput->precision = pSqlObjList[0]->res.precision;
-    
+
     SSchema* schema = NULL;
     if (px->numOfTables > 1) {
       SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES);
@@ -1302,6 +1334,12 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) {
       tfree(pUpQueryInfo);
     }
 
+    if (pQueryInfo->udfCopy) {
+      pQueryInfo->pUdfInfo = taosArrayDestroy(pQueryInfo->pUdfInfo);
+    } else {
+      pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo);
+    }
+
     freeQueryInfoImpl(pQueryInfo);
     clearAllTableMetaInfo(pQueryInfo, removeMeta);
 
@@ -1535,6 +1573,47 @@ void*  tscDestroyBlockArrayList(SArray* pDataBlockList) {
   return NULL;
 }
 
+
+void freeUdfInfo(SUdfInfo* pUdfInfo) {
+  if (pUdfInfo == NULL) {
+    return;
+  }
+
+  if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
+    (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
+  }
+
+  tfree(pUdfInfo->name);
+
+  if (pUdfInfo->path) {
+    unlink(pUdfInfo->path);
+  }
+
+  tfree(pUdfInfo->path);
+
+  tfree(pUdfInfo->content);
+
+  taosCloseDll(pUdfInfo->handle);
+}
+
+
+void*  tscDestroyUdfArrayList(SArray* pUdfList) {
+  if (pUdfList == NULL) {
+    return NULL;
+  }
+
+  size_t size = taosArrayGetSize(pUdfList);
+  for (int32_t i = 0; i < size; i++) {
+    SUdfInfo* udf = taosArrayGet(pUdfList, i);
+    freeUdfInfo(udf);
+  }
+
+  taosArrayDestroy(pUdfList);
+  return NULL;
+}
+
+
+
 void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) {
   if (pBlockHashTable == NULL) {
     return NULL;
@@ -1782,7 +1861,7 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB
 }
 
 int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) {
-  const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); 
+  const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
 
   void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
   SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
@@ -2126,7 +2205,7 @@ void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArr
         assert(pfield->pExpr->pExpr != NULL);
         p.pExpr = calloc(1, sizeof(SExprInfo));
         tscExprAssign(p.pExpr, pfield->pExpr);
-      }    
+      }
 
       taosArrayPush(pFieldInfo->internalField, &p);
     }
@@ -2987,7 +3066,7 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
   pQueryInfo->fillVal        = NULL;
   pQueryInfo->numOfFillVal   = 0;;
   pQueryInfo->clauseLimit    = pSrc->clauseLimit;
-  pQueryInfo->prjOffset      = pSrc->prjOffset;  
+  pQueryInfo->prjOffset      = pSrc->prjOffset;
   pQueryInfo->numOfTables    = 0;
   pQueryInfo->window         = pSrc->window;
   pQueryInfo->sessionWindow  = pSrc->sessionWindow;
@@ -3058,6 +3137,14 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
     tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables);
   }
 
+  SArray *pUdfInfo = NULL;
+  if (pSrc->pUdfInfo) {
+    pUdfInfo = taosArrayDup(pSrc->pUdfInfo);
+  }
+
+  pQueryInfo->pUdfInfo = pUdfInfo;
+  pQueryInfo->udfCopy = true;
+
   _error:
   return code;
 }
@@ -3104,7 +3191,9 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
     info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
   }
 
-  info->itemList = taosArrayDup(pInfo->itemList);
+  if (pInfo->itemList) {
+    info->itemList = taosArrayDup(pInfo->itemList);
+  }
 }
 
 SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) {
@@ -3355,6 +3444,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
 
   SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd);
 
+  if (pQueryInfo->pUdfInfo) {
+    pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo);
+    pNewQueryInfo->udfCopy = true;
+  }
+
   pNewQueryInfo->command = pQueryInfo->command;
   pnCmd->active = pNewQueryInfo;
 
@@ -3374,6 +3468,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
   pNewQueryInfo->numOfTables = 0;
   pNewQueryInfo->pTableMetaInfo = NULL;
   pNewQueryInfo->bufLen = pQueryInfo->bufLen;
+
   pNewQueryInfo->buf = malloc(pQueryInfo->bufLen);
   if (pNewQueryInfo->buf == NULL) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -4216,7 +4311,7 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI
 
       int32_t inter = 0;
       getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType,
-          &pse->resBytes, &inter, 0, false);
+          &pse->resBytes, &inter, 0, false, NULL);
       pse->colType  = pse->resType;
       pse->colBytes = pse->resBytes;
 
@@ -4283,8 +4378,14 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu
         functionId = TSDB_FUNC_STDDEV;
       }
 
+      SUdfInfo* pUdfInfo = NULL;
+
+      if (functionId < 0) {
+        pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
+      }
+
       getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, &pse->resBytes, &inter,
-                        0, false);
+                        0, false, pUdfInfo);
     }
   }
 
@@ -4360,7 +4461,8 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt
   pQueryAttr->order             = pQueryInfo->order;
   pQueryAttr->fillType          = pQueryInfo->fillType;
   pQueryAttr->havingNum         = pQueryInfo->havingFieldNum;
-  
+  pQueryAttr->pUdfInfo          = pQueryInfo->pUdfInfo;
+
   if (pQueryInfo->order.order == TSDB_ORDER_ASC) {   // TODO refactor
     pQueryAttr->window = pQueryInfo->window;
   } else {

src/common/inc/tcmdtype.h

@@ -41,8 +41,10 @@ enum {
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" )  
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" ) 
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" )
@@ -74,6 +76,7 @@ enum {
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_FUNC, "retrieve-function" )
 
   // SQL below for client local 
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" ) 

src/common/inc/tdataformat.h

@@ -31,6 +31,14 @@ extern "C" {
     memcpy(varDataVal(x), (str), __len);          \
   } while (0);
 
+#define STR_TO_NET_VARSTR(x, str)                   \
+    do {                                            \
+      VarDataLenT __len = (VarDataLenT)strlen(str); \
+      *(VarDataLenT *)(x) = htons(__len);           \
+      memcpy(varDataVal(x), (str), __len);          \
+    } while (0);
+
+
 #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)                         \
   do {                                                                    \
     char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \

go @ 7a26c432

-Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4
+Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5

grafanaplugin @ 23cf0b93

-Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa
+Subproject commit 23cf0b933510af5e87b61bace0f6ff2f06d8eaac

hivemq-tdengine-extension @ b62a26ec

-Subproject commit ce5201014136503d34fecbd56494b67b4961056c
+Subproject commit b62a26ecc164a310104df57691691b237e091c89

src/dnode/CMakeLists.txt

@@ -18,7 +18,7 @@ ELSE ()
 ENDIF ()
 
 ADD_EXECUTABLE(taosd ${SRC})
-TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync ${LINK_JEMALLOC})
+TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC})
 
 IF (TD_SOMODE_STATIC)
   TARGET_LINK_LIBRARIES(taosd taos_static)

src/dnode/src/dnodeMain.c

@@ -40,6 +40,7 @@
 #include "dnodeShell.h"
 #include "dnodeTelemetry.h"
 #include "module.h"
+#include "qScript.h"
 #include "mnode.h"
 
 #if !defined(_MODULE) || !defined(_TD_LINUX)
@@ -84,6 +85,7 @@ static SStep tsDnodeSteps[] = {
   {"dnode-shell",     dnodeInitShell,      dnodeCleanupShell},
   {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer},
   {"dnode-telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
+  {"dnode-script",    scriptEnvPoolInit,   scriptEnvPoolCleanup},
 };
 
 static SStep tsDnodeCompactSteps[] = {
@@ -266,7 +268,7 @@ static int32_t dnodeInitStorage() {
       return -1;
     }
   }
-  //TODO(dengyihao): no need to init here 
+  //TODO(dengyihao): no need to init here
   if (dnodeCreateDir(tsMnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
    return -1;

src/dnode/src/dnodeShell.c

@@ -48,9 +48,11 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB]     = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]     = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]    = dnodeDispatchToMWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
@@ -72,6 +74,7 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMReadQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = dnodeDispatchToMReadQueue;
 
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]   = dnodeSendStartupStep;
 

src/inc/query.h

@@ -28,7 +28,7 @@ typedef void* qinfo_t;
  * @param qinfo
  * @return
  */
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t *qId);
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId);
 
 
 /**

src/inc/taosdef.h

@@ -186,6 +186,10 @@ do { \
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       193     // it is a null-terminated string
 #define TSDB_DB_NAME_LEN          33
+#define TSDB_FUNC_NAME_LEN        65
+#define TSDB_FUNC_CODE_LEN        (65535 - 512)
+#define TSDB_FUNC_BUF_SIZE        512
+#define TSDB_TYPE_STR_MAX_LEN     32
 #define TSDB_TABLE_FNAME_LEN      (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
@@ -332,6 +336,10 @@ do { \
 #define TSDB_QUERY_TYPE_NON_TYPE               0x00u     // none type
 #define TSDB_QUERY_TYPE_FREE_RESOURCE          0x01u     // free qhandle at vnode
 
+#define TSDB_UDF_TYPE_SCALAR       1
+#define TSDB_UDF_TYPE_AGGREGATE    2
+
+
 /*
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag

src/inc/taoserror.h

@@ -100,6 +100,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TSC_DB_NOT_SELECTED           TAOS_DEF_ERROR_CODE(0, 0x0217)  //"Database not specified or available")
 #define TSDB_CODE_TSC_INVALID_TABLE_NAME        TAOS_DEF_ERROR_CODE(0, 0x0218)  //"Table does not exist")
 #define TSDB_CODE_TSC_EXCEED_SQL_LIMIT          TAOS_DEF_ERROR_CODE(0, 0x0219)  //"SQL statement too long check maxSQLLength config")
+#define TSDB_CODE_TSC_FILE_EMPTY                TAOS_DEF_ERROR_CODE(0, 0x021A)  //"File is empty")
 
 // mnode
 #define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0300)  //"Message not processed")
@@ -174,6 +175,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_STABLE_NAME       TAOS_DEF_ERROR_CODE(0, 0x036D)  //"Super table does not exist")
 #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG  TAOS_DEF_ERROR_CODE(0, 0x036E)  //"Invalid create table message")
 
+#define TSDB_CODE_MND_INVALID_FUNC_NAME         TAOS_DEF_ERROR_CODE(0, 0x0370)  //"Invalid func name")
+#define TSDB_CODE_MND_INVALID_FUNC_LEN          TAOS_DEF_ERROR_CODE(0, 0x0371)  //"Invalid func length")
+#define TSDB_CODE_MND_INVALID_FUNC_CODE         TAOS_DEF_ERROR_CODE(0, 0x0372)  //"Invalid func code")
+#define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x0373)  //"Func already exists")
+#define TSDB_CODE_MND_INVALID_FUNC              TAOS_DEF_ERROR_CODE(0, 0x0374)  //"Invalid func")
+#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE      TAOS_DEF_ERROR_CODE(0, 0x0375)  //"Invalid func bufSize")
+
 #define TSDB_CODE_MND_DB_NOT_SELECTED           TAOS_DEF_ERROR_CODE(0, 0x0380)  //"Database not specified or available")
 #define TSDB_CODE_MND_DB_ALREADY_EXIST          TAOS_DEF_ERROR_CODE(0, 0x0381)  //"Database already exists")
 #define TSDB_CODE_MND_INVALID_DB_OPTION         TAOS_DEF_ERROR_CODE(0, 0x0382)  //"Invalid database options")
@@ -261,6 +269,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW       TAOS_DEF_ERROR_CODE(0, 0x070A)  //"Too many time window in query")
 #define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER         TAOS_DEF_ERROR_CODE(0, 0x070B)  //"Query buffer limit has reached")
 #define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistency in replica")
+#define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070D)  //"System error")
 
 
 // grant

src/inc/taosmsg.h

@@ -77,8 +77,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_USER, "drop-user" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DNODE, "create-dnode" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DNODE, "drop-dnode" )   
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )	  
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )	 
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_FUNCTION, "create-function" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_FUNCTION, "drop-function" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SYNC_DB, "sync-db-replica" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" )
@@ -96,7 +98,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_STREAM, "kill-stream" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_CONN, "kill-conn" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONFIG_DNODE, "cm-config-dnode" ) 
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_HEARTBEAT, "heartbeat" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY8, "dummy8" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, "retrieve-func" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY10, "dummy10" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY11, "dummy11" )
@@ -153,6 +155,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_STREAMTABLES,
   TSDB_MGMT_TABLE_CLUSTER,
   TSDB_MGMT_TABLE_TP,
+  TSDB_MGMT_TABLE_FUNCTION,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -507,6 +510,9 @@ typedef struct {
   int32_t     prevResultLen;    // previous result length
   int32_t     numOfOperator;
   int32_t     tableScanOperator;// table scan operator. -1 means no scan operator
+  int32_t     udfNum;           // number of udf function
+  int32_t     udfContentOffset;
+  int32_t     udfContentLen;
   SColumnInfo tableCols[];
 } SQueryTableMsg;
 
@@ -571,6 +577,41 @@ typedef struct {
   int8_t   reserve[5];
 } SCreateDbMsg, SAlterDbMsg;
 
+typedef struct {
+  char     name[TSDB_FUNC_NAME_LEN];
+  char     path[PATH_MAX];
+  int32_t  funcType;
+  uint8_t  outputType;
+  int16_t  outputLen;
+  int32_t  bufSize;
+  int32_t  codeLen;
+  char     code[];
+} SCreateFuncMsg;
+
+typedef struct {
+  int32_t num;
+  char    name[];
+} SRetrieveFuncMsg;
+
+typedef struct {
+  char    name[TSDB_FUNC_NAME_LEN];
+  int32_t funcType;
+  int8_t  resType;
+  int16_t resBytes;
+  int32_t bufSize;
+  int32_t len;
+  char    content[];
+} SFunctionInfoMsg;
+
+typedef struct {
+  int32_t num;
+  char    content[];
+} SUdfFuncMsg;
+
+typedef struct {
+  char     name[TSDB_FUNC_NAME_LEN];
+} SDropFuncMsg;
+
 typedef struct {
   char    db[TSDB_TABLE_FNAME_LEN];
   uint8_t ignoreNotExists;
@@ -712,6 +753,7 @@ typedef struct {
 typedef struct {
   int32_t numOfVgroups;
   int32_t numOfTables;
+  int32_t numOfUdfs;
   char    tableNames[];
 } SMultiTableInfoMsg;
 
@@ -760,12 +802,13 @@ typedef struct STableMetaMsg {
 } STableMetaMsg;
 
 typedef struct SMultiTableMeta {
-  int32_t  numOfTables;
-  int32_t  numOfVgroup;
-  uint32_t contLen:31;
-  uint8_t  compressed:1;      // denote if compressed or not
-  uint32_t rawLen;            // size before compress
-  char     meta[];
+  int32_t       numOfTables;
+  int32_t       numOfVgroup;
+  int32_t       numOfUdf;
+  int32_t       contLen;
+  uint8_t       compressed;      // denote if compressed or not
+  uint32_t      rawLen;          // size before compress
+  char          meta[];
 } SMultiTableMeta;
 
 typedef struct {

src/inc/ttokendef.h

@@ -62,149 +62,154 @@
 #define TK_SHOW                            44
 #define TK_DATABASES                       45
 #define TK_TOPICS                          46
-#define TK_MNODES                          47
-#define TK_DNODES                          48
-#define TK_ACCOUNTS                        49
-#define TK_USERS                           50
-#define TK_MODULES                         51
-#define TK_QUERIES                         52
-#define TK_CONNECTIONS                     53
-#define TK_STREAMS                         54
-#define TK_VARIABLES                       55
-#define TK_SCORES                          56
-#define TK_GRANTS                          57
-#define TK_VNODES                          58
-#define TK_IPTOKEN                         59
-#define TK_DOT                             60
-#define TK_CREATE                          61
-#define TK_TABLE                           62
-#define TK_STABLE                          63
-#define TK_DATABASE                        64
-#define TK_TABLES                          65
-#define TK_STABLES                         66
-#define TK_VGROUPS                         67
-#define TK_DROP                            68
-#define TK_TOPIC                           69
-#define TK_DNODE                           70
-#define TK_USER                            71
-#define TK_ACCOUNT                         72
-#define TK_USE                             73
-#define TK_DESCRIBE                        74
-#define TK_ALTER                           75
-#define TK_PASS                            76
-#define TK_PRIVILEGE                       77
-#define TK_LOCAL                           78
-#define TK_COMPACT                         79
-#define TK_LP                              80
-#define TK_RP                              81
-#define TK_IF                              82
-#define TK_EXISTS                          83
-#define TK_PPS                             84
-#define TK_TSERIES                         85
-#define TK_DBS                             86
-#define TK_STORAGE                         87
-#define TK_QTIME                           88
-#define TK_CONNS                           89
-#define TK_STATE                           90
-#define TK_COMMA                           91
-#define TK_KEEP                            92
-#define TK_CACHE                           93
-#define TK_REPLICA                         94
-#define TK_QUORUM                          95
-#define TK_DAYS                            96
-#define TK_MINROWS                         97
-#define TK_MAXROWS                         98
-#define TK_BLOCKS                          99
-#define TK_CTIME                          100
-#define TK_WAL                            101
-#define TK_FSYNC                          102
-#define TK_COMP                           103
-#define TK_PRECISION                      104
-#define TK_UPDATE                         105
-#define TK_CACHELAST                      106
-#define TK_PARTITIONS                     107
-#define TK_UNSIGNED                       108
-#define TK_TAGS                           109
-#define TK_USING                          110
-#define TK_AS                             111
-#define TK_NULL                           112
-#define TK_NOW                            113
-#define TK_SELECT                         114
-#define TK_UNION                          115
-#define TK_ALL                            116
-#define TK_DISTINCT                       117
-#define TK_FROM                           118
-#define TK_VARIABLE                       119
-#define TK_INTERVAL                       120
-#define TK_SESSION                        121
-#define TK_STATE_WINDOW                   122
-#define TK_FILL                           123
-#define TK_SLIDING                        124
-#define TK_ORDER                          125
-#define TK_BY                             126
-#define TK_ASC                            127
-#define TK_DESC                           128
-#define TK_GROUP                          129
-#define TK_HAVING                         130
-#define TK_LIMIT                          131
-#define TK_OFFSET                         132
-#define TK_SLIMIT                         133
-#define TK_SOFFSET                        134
-#define TK_WHERE                          135
-#define TK_RESET                          136
-#define TK_QUERY                          137
-#define TK_SYNCDB                         138
-#define TK_ADD                            139
-#define TK_COLUMN                         140
-#define TK_MODIFY                         141
-#define TK_TAG                            142
-#define TK_CHANGE                         143
-#define TK_SET                            144
-#define TK_KILL                           145
-#define TK_CONNECTION                     146
-#define TK_STREAM                         147
-#define TK_COLON                          148
-#define TK_ABORT                          149
-#define TK_AFTER                          150
-#define TK_ATTACH                         151
-#define TK_BEFORE                         152
-#define TK_BEGIN                          153
-#define TK_CASCADE                        154
-#define TK_CLUSTER                        155
-#define TK_CONFLICT                       156
-#define TK_COPY                           157
-#define TK_DEFERRED                       158
-#define TK_DELIMITERS                     159
-#define TK_DETACH                         160
-#define TK_EACH                           161
-#define TK_END                            162
-#define TK_EXPLAIN                        163
-#define TK_FAIL                           164
-#define TK_FOR                            165
-#define TK_IGNORE                         166
-#define TK_IMMEDIATE                      167
-#define TK_INITIALLY                      168
-#define TK_INSTEAD                        169
-#define TK_MATCH                          170
-#define TK_KEY                            171
-#define TK_OF                             172
-#define TK_RAISE                          173
-#define TK_REPLACE                        174
-#define TK_RESTRICT                       175
-#define TK_ROW                            176
-#define TK_STATEMENT                      177
-#define TK_TRIGGER                        178
-#define TK_VIEW                           179
-#define TK_SEMI                           180
-#define TK_NONE                           181
-#define TK_PREV                           182
-#define TK_LINEAR                         183
-#define TK_IMPORT                         184
-#define TK_TBNAME                         185
-#define TK_JOIN                           186
-#define TK_INSERT                         187
-#define TK_INTO                           188
-#define TK_VALUES                         189
+#define TK_FUNCTIONS                       47
+#define TK_MNODES                          48
+#define TK_DNODES                          49
+#define TK_ACCOUNTS                        50
+#define TK_USERS                           51
+#define TK_MODULES                         52
+#define TK_QUERIES                         53
+#define TK_CONNECTIONS                     54
+#define TK_STREAMS                         55
+#define TK_VARIABLES                       56
+#define TK_SCORES                          57
+#define TK_GRANTS                          58
+#define TK_VNODES                          59
+#define TK_IPTOKEN                         60
+#define TK_DOT                             61
+#define TK_CREATE                          62
+#define TK_TABLE                           63
+#define TK_STABLE                          64
+#define TK_DATABASE                        65
+#define TK_TABLES                          66
+#define TK_STABLES                         67
+#define TK_VGROUPS                         68
+#define TK_DROP                            69
+#define TK_TOPIC                           70
+#define TK_FUNCTION                        71
+#define TK_DNODE                           72
+#define TK_USER                            73
+#define TK_ACCOUNT                         74
+#define TK_USE                             75
+#define TK_DESCRIBE                        76
+#define TK_ALTER                           77
+#define TK_PASS                            78
+#define TK_PRIVILEGE                       79
+#define TK_LOCAL                           80
+#define TK_COMPACT                         81
+#define TK_LP                              82
+#define TK_RP                              83
+#define TK_IF                              84
+#define TK_EXISTS                          85
+#define TK_AS                              86
+#define TK_OUTPUTTYPE                      87
+#define TK_AGGREGATE                       88
+#define TK_BUFSIZE                         89
+#define TK_PPS                             90
+#define TK_TSERIES                         91
+#define TK_DBS                             92
+#define TK_STORAGE                         93
+#define TK_QTIME                           94
+#define TK_CONNS                           95
+#define TK_STATE                           96
+#define TK_COMMA                           97
+#define TK_KEEP                            98
+#define TK_CACHE                           99
+#define TK_REPLICA                        100
+#define TK_QUORUM                         101
+#define TK_DAYS                           102
+#define TK_MINROWS                        103
+#define TK_MAXROWS                        104
+#define TK_BLOCKS                         105
+#define TK_CTIME                          106
+#define TK_WAL                            107
+#define TK_FSYNC                          108
+#define TK_COMP                           109
+#define TK_PRECISION                      110
+#define TK_UPDATE                         111
+#define TK_CACHELAST                      112
+#define TK_PARTITIONS                     113
+#define TK_UNSIGNED                       114
+#define TK_TAGS                           115
+#define TK_USING                          116
+#define TK_NULL                           117
+#define TK_NOW                            118
+#define TK_SELECT                         119
+#define TK_UNION                          120
+#define TK_ALL                            121
+#define TK_DISTINCT                       122
+#define TK_FROM                           123
+#define TK_VARIABLE                       124
+#define TK_INTERVAL                       125
+#define TK_SESSION                        126
+#define TK_STATE_WINDOW                   127
+#define TK_FILL                           128
+#define TK_SLIDING                        129
+#define TK_ORDER                          130
+#define TK_BY                             131
+#define TK_ASC                            132
+#define TK_DESC                           133
+#define TK_GROUP                          134
+#define TK_HAVING                         135
+#define TK_LIMIT                          136
+#define TK_OFFSET                         137
+#define TK_SLIMIT                         138
+#define TK_SOFFSET                        139
+#define TK_WHERE                          140
+#define TK_RESET                          141
+#define TK_QUERY                          142
+#define TK_SYNCDB                         143
+#define TK_ADD                            144
+#define TK_COLUMN                         145
+#define TK_MODIFY                         146
+#define TK_TAG                            147
+#define TK_CHANGE                         148
+#define TK_SET                            149
+#define TK_KILL                           150
+#define TK_CONNECTION                     151
+#define TK_STREAM                         152
+#define TK_COLON                          153
+#define TK_ABORT                          154
+#define TK_AFTER                          155
+#define TK_ATTACH                         156
+#define TK_BEFORE                         157
+#define TK_BEGIN                          158
+#define TK_CASCADE                        159
+#define TK_CLUSTER                        160
+#define TK_CONFLICT                       161
+#define TK_COPY                           162
+#define TK_DEFERRED                       163
+#define TK_DELIMITERS                     164
+#define TK_DETACH                         165
+#define TK_EACH                           166
+#define TK_END                            167
+#define TK_EXPLAIN                        168
+#define TK_FAIL                           169
+#define TK_FOR                            170
+#define TK_IGNORE                         171
+#define TK_IMMEDIATE                      172
+#define TK_INITIALLY                      173
+#define TK_INSTEAD                        174
+#define TK_MATCH                          175
+#define TK_KEY                            176
+#define TK_OF                             177
+#define TK_RAISE                          178
+#define TK_REPLACE                        179
+#define TK_RESTRICT                       180
+#define TK_ROW                            181
+#define TK_STATEMENT                      182
+#define TK_TRIGGER                        183
+#define TK_VIEW                           184
+#define TK_SEMI                           185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_IMPORT                         189
+#define TK_TBNAME                         190
+#define TK_JOIN                           191
+#define TK_INSERT                         192
+#define TK_INTO                           193
+#define TK_VALUES                         194
 
 
 #define TK_SPACE                          300

src/inc/ttype.h

@@ -28,6 +28,10 @@ typedef struct tstr {
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
 #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
+#define varDataNetLen(v)       (htons(((VarDataLenT *)(v))[0]))
+#define varDataNetTLen(v)      (sizeof(VarDataLenT) + varDataNetLen(v))
+
+
 // this data type is internally used only in 'in' query to hold the values
 #define TSDB_DATA_TYPE_ARRAY      (1000)
 

src/kit/shell/CMakeLists.txt

@@ -19,9 +19,9 @@ ELSE ()
 ENDIF ()
 
   IF (TD_SOMODE_STATIC)
-      TARGET_LINK_LIBRARIES(shell taos_static ${LINK_JEMALLOC})
+      TARGET_LINK_LIBRARIES(shell taos_static lua ${LINK_JEMALLOC})
   ELSE ()
-      TARGET_LINK_LIBRARIES(shell taos ${LINK_JEMALLOC})
+      TARGET_LINK_LIBRARIES(shell taos lua ${LINK_JEMALLOC})
   ENDIF ()
 
   SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

src/kit/taosdemo/CMakeLists.txt

@@ -67,7 +67,7 @@ IF (TD_LINUX)
   ADD_EXECUTABLE(taosdemo ${SRC})
 
   IF (TD_SOMODE_STATIC)
-      TARGET_LINK_LIBRARIES(taosdemo taos_static cJson ${LINK_JEMALLOC})
+      TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua ${LINK_JEMALLOC})
   ELSE ()
       TARGET_LINK_LIBRARIES(taosdemo taos cJson ${LINK_JEMALLOC})
   ENDIF ()
@@ -76,9 +76,9 @@ ELSEIF (TD_WINDOWS)
   ADD_EXECUTABLE(taosdemo ${SRC})
   SET_SOURCE_FILES_PROPERTIES(./taosdemo.c PROPERTIES COMPILE_FLAGS -w)
   IF (TD_SOMODE_STATIC)
-    TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
+    TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
   ELSE ()
-    TARGET_LINK_LIBRARIES(taosdemo taos cJson)
+    TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   # missing a few dependencies, such as <argp.h>
@@ -86,9 +86,9 @@ ELSEIF (TD_DARWIN)
   ADD_EXECUTABLE(taosdemo ${SRC})
   
   IF (TD_SOMODE_STATIC)
-    TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
+    TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
   ELSE ()
-    TARGET_LINK_LIBRARIES(taosdemo taos cJson)
+    TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
   ENDIF ()
 ENDIF ()
 

src/mnode/inc/mnodeDef.h

@@ -214,6 +214,23 @@ typedef struct SUserObj {
   struct SAcctObj * pAcct;
 } SUserObj;
 
+typedef struct SFuncObj {
+  char              name[TSDB_FUNC_NAME_LEN];
+  char              path[128];
+  int32_t           contLen;
+  char              cont[TSDB_FUNC_CODE_LEN];
+  int32_t           funcType;
+  int32_t           bufSize;
+  int64_t           createdTime;
+  uint8_t           resType;
+  int16_t           resBytes;
+  int64_t           sig;         // partial md5 sign
+  int16_t           type;        // [lua script|so|js]
+  int8_t            reserved[64];
+  int8_t            updateEnd[4];
+  int32_t           refCount;
+} SFuncObj;
+
 typedef struct {
   int64_t totalStorage;  // Total storage wrtten from this account
   int64_t compStorage;   // Compressed storage on disk
@@ -258,7 +275,7 @@ typedef struct {
   void *   pIter;
   void **  ppShow;
   int16_t  offset[TSDB_MAX_COLUMNS];
-  int16_t  bytes[TSDB_MAX_COLUMNS];
+  int32_t  bytes[TSDB_MAX_COLUMNS];
   int32_t  numOfReads;
   int8_t   maxReplica;
   int8_t   reserved0[1];

src/mnode/inc/mnodeFunc.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_FUNC_H
+#define TDENGINE_MNODE_FUNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeDef.h"
+
+int32_t   mnodeInitFuncs();
+void      mnodeCleanupFuncs();
+
+SFuncObj *mnodeGetFunc(char *name);
+void *    mnodeGetNextFunc(void *pIter, SFuncObj **pFunc);
+void      mnodeCancelGetNextFunc(void *pIter);
+
+void      mnodeIncFuncRef(SFuncObj *pFunc);
+void      mnodeDecFuncRef(SFuncObj *pFunc);
+
+int32_t   mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *code, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mnodeSdb.h

@@ -33,7 +33,8 @@ typedef enum {
   SDB_TABLE_VGROUP  = 6,
   SDB_TABLE_STABLE  = 7,
   SDB_TABLE_CTABLE  = 8,
-  SDB_TABLE_MAX     = 9
+  SDB_TABLE_FUNC    = 9,
+  SDB_TABLE_MAX     = 10
 } ESdbTable;
 
 typedef enum {
@@ -112,4 +113,4 @@ int32_t mnodeCompactWal();
 }
 #endif
 
-#endif
+#endif
\ No newline at end of file

src/mnode/src/mnodeDb.c

@@ -554,6 +554,9 @@ void mnodeCleanupDbs() {
   tsDbSdb = NULL;
 }
 
+
+
+
 static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 

src/mnode/src/mnodeFunc.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "trpc.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "tgrant.h"
+#include "tdataformat.h"
+#include "tkey.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeAcct.h"
+#include "mnodeUser.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeFunc.h"
+#include "mnodeWrite.h"
+#include "mnodeRead.h"
+#include "mnodePeer.h"
+
+int64_t        tsFuncRid = -1;
+static void *  tsFuncSdb = NULL;
+static int32_t tsFuncUpdateSize = 0;
+static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg);
+static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg);
+static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg);
+
+static int32_t mnodeFuncActionDestroy(SSdbRow *pRow) {
+  tfree(pRow->pObj);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionInsert(SSdbRow *pRow) {
+  SFuncObj *pFunc = pRow->pObj;
+
+  mTrace("func:%s, contLen: %d, insert into sdb", pFunc->name, pFunc->contLen);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionDelete(SSdbRow *pRow) {
+  SFuncObj *pFunc = pRow->pObj;
+
+  mTrace("func:%s, length: %d, delete from sdb", pFunc->name, pFunc->contLen);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionUpdate(SSdbRow *pRow) {
+  SFuncObj *pFunc = pRow->pObj;
+
+  SFuncObj *pSaved = mnodeGetFunc(pFunc->name);
+  if (pFunc != pSaved) {
+    memcpy(pSaved, pFunc, tsFuncUpdateSize);
+    free(pFunc);
+  }
+
+  mnodeDecFuncRef(pSaved);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionEncode(SSdbRow *pRow) {
+  SFuncObj *pFunc = pRow->pObj;
+
+  memcpy(pRow->rowData, pFunc, tsFuncUpdateSize);
+  pRow->rowSize = tsFuncUpdateSize;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionDecode(SSdbRow *pRow) {
+  SFuncObj *pFunc = (SFuncObj *)calloc(1, sizeof(SFuncObj));
+  if (pFunc == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
+
+  memcpy(pFunc, pRow->rowData, tsFuncUpdateSize);
+  pRow->pObj = pFunc;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mnodeFuncActionRestored() {
+  int64_t numOfRows = sdbGetNumOfRows(tsFuncSdb);
+
+  if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
+    mInfo("dnode first deploy, func restored.");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mnodeInitFuncs() {
+  SFuncObj tObj;
+  tsFuncUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
+
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_FUNC,
+    .name         = "funcs",
+    .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
+    .maxRowSize   = tsFuncUpdateSize,
+    .refCountPos  = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
+    .keyType      = SDB_KEY_STRING,
+    .fpInsert     = mnodeFuncActionInsert,
+    .fpDelete     = mnodeFuncActionDelete,
+    .fpUpdate     = mnodeFuncActionUpdate,
+    .fpEncode     = mnodeFuncActionEncode,
+    .fpDecode     = mnodeFuncActionDecode,
+    .fpDestroy    = mnodeFuncActionDestroy,
+    .fpRestored   = mnodeFuncActionRestored
+  };
+
+  tsFuncRid = sdbOpenTable(&desc);
+  tsFuncSdb = sdbGetTableByRid(tsFuncRid);
+  if (tsFuncSdb == NULL) {
+    mError("table:%s, failed to create hash", desc.name);
+    return -1;
+  }
+
+  mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_CREATE_FUNCTION, mnodeProcessCreateFuncMsg);
+  mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_FUNCTION, mnodeProcessDropFuncMsg);
+  mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, mnodeProcessRetrieveFuncImplMsg);
+
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeGetFuncMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeRetrieveFuncs);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeCancelGetNextFunc);
+
+  mDebug("table:%s, hash is created", desc.name);
+
+  return 0;
+}
+
+void mnodeCleanupFuncs() {
+  sdbCloseTable(tsFuncRid);
+  tsFuncSdb = NULL;
+}
+
+SFuncObj *mnodeGetFunc(char *name) {
+  return (SFuncObj *)sdbGetRow(tsFuncSdb, name);
+}
+
+void *mnodeGetNextFunc(void *pIter, SFuncObj **pFunc) {
+  return sdbFetchRow(tsFuncSdb, pIter, (void **)pFunc);
+}
+
+void mnodeCancelGetNextFunc(void *pIter) {
+ sdbFreeIter(tsFuncSdb, pIter);
+}
+
+void mnodeIncFuncRef(SFuncObj *pFunc) {
+  sdbIncRef(tsFuncSdb, pFunc);
+}
+
+void mnodeDecFuncRef(SFuncObj *pFunc) {
+  sdbDecRef(tsFuncSdb, pFunc);
+}
+/*
+static int32_t mnodeUpdateFunc(SFuncObj *pFunc, void *pMsg) {
+  SSdbRow row = {
+    .type   = SDB_OPER_GLOBAL,
+    .pTable = tsFuncSdb,
+    .pObj   = pFunc,
+    .pMsg   = pMsg
+  };
+
+  int32_t code = sdbUpdateRow(&row);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("func:%s, failed to alter by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
+  } else {
+    mLInfo("func:%s, is altered by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
+  }
+
+  return code;
+}
+*/
+int32_t mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *codeScript, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg) {
+  if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_GRANT_EXPIRED;
+  }
+
+  if (!pMsg->pUser->writeAuth) {
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  code = grantCheck(TSDB_GRANT_USER);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  if (name[0] == 0) {
+    return TSDB_CODE_MND_INVALID_FUNC_NAME;
+  }
+
+  if (codeScript[0] == 0) {
+    return TSDB_CODE_MND_INVALID_FUNC_CODE;
+  }
+
+  if (codeLen < 0 || codeLen > TSDB_FUNC_CODE_LEN) {
+    return TSDB_CODE_MND_INVALID_FUNC_CODE;
+  }
+
+  if (bufSize < 0 || bufSize > TSDB_FUNC_BUF_SIZE) {
+    return TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
+  }
+
+  SFuncObj *pFunc = mnodeGetFunc(name);
+  if (pFunc != NULL) {
+    mDebug("func:%s, is already there", name);
+    mnodeDecFuncRef(pFunc);
+    return TSDB_CODE_MND_FUNC_ALREADY_EXIST;
+  }
+
+  pFunc = calloc(1, sizeof(SFuncObj));
+  tstrncpy(pFunc->name, name, TSDB_FUNC_NAME_LEN);
+  tstrncpy(pFunc->path, path, tListLen(pFunc->path));
+  memcpy(pFunc->cont, codeScript, codeLen);
+  pFunc->contLen     = codeLen;
+  pFunc->createdTime = taosGetTimestampMs();
+  pFunc->resType     = outputType;
+  pFunc->resBytes    = outputLen;
+  pFunc->funcType    = funcType;
+  pFunc->bufSize     = bufSize;
+  pFunc->sig  = 0;
+  pFunc->type = 1; //lua script, refactor
+
+  SSdbRow row = {
+    .type     = SDB_OPER_GLOBAL,
+    .pTable   = tsFuncSdb,
+    .pObj     = pFunc,
+    .rowSize  = sizeof(SFuncObj),
+    .pMsg     = pMsg
+  };
+
+  code = sdbInsertRow(&row);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("func:%s, failed to create by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
+    tfree(pFunc);
+  } else {
+    mLInfo("func:%s, is created by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
+  }
+
+  return code;
+}
+
+static int32_t mnodeDropFunc(SFuncObj *pFunc, void *pMsg) {
+  SSdbRow row = {
+    .type   = SDB_OPER_GLOBAL,
+    .pTable = tsFuncSdb,
+    .pObj   = pFunc,
+    .pMsg   = pMsg
+  };
+
+  int32_t code = sdbDeleteRow(&row);
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("func:%s, failed to drop by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
+  } else {
+    mLInfo("func:%s, is dropped by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
+  }
+
+  return code;
+}
+
+static int32_t mnodeGetFuncsNum() {
+  return (int32_t)sdbGetNumOfRows(tsFuncSdb);
+}
+
+static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) {
+    return TSDB_CODE_MND_NO_USER_FROM_CONN;
+  }
+
+  int32_t cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "name");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "path");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "aggregate");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "outputtype");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 8;
+  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
+  strcpy(pSchema[cols].name, "create_time");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "code_len");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "bufsize");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  strcpy(pMeta->tableFname, "show funcs");
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetFuncsNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static void* mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) {
+  char *msg = "unknown";
+  if (type >= sizeof(tDataTypes)/sizeof(tDataTypes[0])) {
+    return msg;
+  }
+  
+  if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
+    int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len;
+    
+    snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes/4 : bytes);
+    buf[buflen - 1] = 0;
+
+    return buf;
+  }
+  
+  return tDataTypes[type].name;
+}
+
+static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t   numOfRows = 0;
+  SFuncObj *pFunc    = NULL;
+  int32_t   cols      = 0;
+  char     *pWrite;
+  char      buf[TSDB_TYPE_STR_MAX_LEN];
+
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextFunc(pShow->pIter, &pFunc);
+    if (pFunc == NULL) break;
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->name, pShow->bytes[cols]);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->path, pShow->bytes[cols]);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pFunc->funcType == TSDB_UDF_TYPE_AGGREGATE ? 1 : 0;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->resType, pFunc->resBytes), pShow->bytes[cols]);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t *)pWrite = pFunc->createdTime;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pFunc->contLen;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pFunc->bufSize;
+    cols++;
+
+    numOfRows++;
+    mnodeDecFuncRef(pFunc);
+  }
+
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
+
+static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg) {
+  SCreateFuncMsg *pCreate    = pMsg->rpcMsg.pCont;
+  pCreate->codeLen       = htonl(pCreate->codeLen);
+  pCreate->outputLen     = htons(pCreate->outputLen);
+  pCreate->funcType      = htonl(pCreate->funcType);
+  pCreate->bufSize       = htonl(pCreate->bufSize);
+
+  return mnodeCreateFunc(pMsg->pUser->pAcct, pCreate->name, pCreate->codeLen, pCreate->code, pCreate->path, pCreate->outputType, pCreate->outputLen, pCreate->funcType, pCreate->bufSize, pMsg);
+}
+
+static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg) {
+  SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont;
+
+  SFuncObj *pFunc = mnodeGetFunc(pDrop->name);
+  if (pFunc == NULL) {
+    return TSDB_CODE_MND_INVALID_FUNC;
+  }
+
+  return mnodeDropFunc(pFunc, pMsg);
+}
+
+static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg) {
+  SRetrieveFuncMsg *pInfo = pMsg->rpcMsg.pCont;
+  pInfo->num = htonl(pInfo->num);
+
+  int32_t t = sizeof(SUdfFuncMsg) + (sizeof(SFunctionInfoMsg) + TSDB_FUNC_CODE_LEN) * pInfo->num + 16384;
+
+  SUdfFuncMsg *pFuncMsg = rpcMallocCont(t);
+  pFuncMsg->num = htonl(pInfo->num);
+  char* pOutput = pFuncMsg->content;
+  tstr* name = (tstr*) pInfo->name;
+
+  for(int32_t i = 0; i < pInfo->num; ++i) {
+    char buf[TSDB_FUNC_NAME_LEN] = {0};
+    tstrncpy(buf, name->data, htons(name->len) + 1);
+
+    SFuncObj* pFuncObj = mnodeGetFunc(buf);
+    if (pFuncObj == NULL) {
+      mError("function %s does not exist", buf);
+      return TSDB_CODE_MND_INVALID_FUNC;
+    }
+    
+    SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) pOutput;
+
+    strcpy(pFuncInfo->name, buf);
+    pFuncInfo->len = htonl(pFuncObj->contLen);
+    memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
+
+    pFuncInfo->funcType = htonl(pFuncObj->funcType);
+    pFuncInfo->resType = pFuncObj->resType;
+    pFuncInfo->resBytes = htons(pFuncObj->resBytes);
+    pFuncInfo->bufSize  = htonl(pFuncObj->bufSize);
+    
+    pOutput += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
+    name =(tstr *)((char *)name + sizeof(*name) + htons(name->len));
+  }
+
+  pMsg->rpcRsp.rsp = pFuncMsg;
+  pMsg->rpcRsp.len = (int32_t)(pOutput - (char*)pFuncMsg);
+  return TSDB_CODE_SUCCESS;
+}

src/mnode/src/mnodeMain.c

@@ -32,6 +32,7 @@
 #include "mnodeSdb.h"
 #include "mnodeVgroup.h"
 #include "mnodeUser.h"
+#include "mnodeFunc.h"
 #include "mnodeTable.h"
 #include "mnodeCluster.h"
 #include "mnodeShow.h"
@@ -46,6 +47,7 @@ static SStep tsMnodeSteps[] = {
   {"cluster", mnodeInitCluster, mnodeCleanupCluster},
   {"accts",   mnodeInitAccts,   mnodeCleanupAccts},
   {"users",   mnodeInitUsers,   mnodeCleanupUsers},
+  {"funcs",   mnodeInitFuncs,   mnodeCleanupFuncs},
   {"dnodes",  mnodeInitDnodes,  mnodeCleanupDnodes},
   {"dbs",     mnodeInitDbs,     mnodeCleanupDbs},
   {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups},

src/mnode/src/mnodeTable.c

@@ -42,6 +42,7 @@
 #include "mnodeWrite.h"
 #include "mnodeRead.h"
 #include "mnodePeer.h"
+#include "mnodeFunc.h"
 
 #define ALTER_CTABLE_RETRY_TIMES  3
 #define CREATE_CTABLE_RETRY_TIMES 10
@@ -2926,6 +2927,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
 
   pInfo->numOfTables  = htonl(pInfo->numOfTables);
   pInfo->numOfVgroups = htonl(pInfo->numOfVgroups);
+  pInfo->numOfUdfs    = htonl(pInfo->numOfUdfs);
 
   int32_t contLen = pMsg->rpcMsg.contLen - sizeof(SMultiTableInfoMsg);
 
@@ -2934,9 +2936,9 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
   char*   str      = strndup(pInfo->tableNames, contLen);
   char**  nameList = strsplit(str, ",", &num);
   SArray* pList    = taosArrayInit(4, POINTER_BYTES);
-  SMultiTableMeta *pMultiMeta = NULL;
 
-  if (num != pInfo->numOfTables + pInfo->numOfVgroups) {
+  SMultiTableMeta *pMultiMeta = NULL;
+  if (num != pInfo->numOfTables + pInfo->numOfVgroups + pInfo->numOfUdfs) {
     mError("msg:%p, app:%p, failed to get multi-tableMeta, msg inconsistent", pMsg, pMsg->rpcMsg.ahandle);
     code = TSDB_CODE_MND_INVALID_TABLE_NAME;
     goto _end;
@@ -3011,8 +3013,10 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
     }
   }
 
+  int32_t tableNum = pInfo->numOfTables + pInfo->numOfVgroups;
+
   // add the additional super table names that needs the vgroup info
-  for(;t < num; ++t) {
+  for(;t < tableNum; ++t) {
     taosArrayPush(pList, &nameList[t]);
   }
 
@@ -3043,6 +3047,36 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
   pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta);
 
   pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
+
+  // add the user-defined-function information
+  for(int32_t i = 0; i < pInfo->numOfUdfs; ++i, ++t) {
+    char buf[TSDB_FUNC_NAME_LEN] = {0};
+    strcpy(buf, nameList[t]);
+
+    SFuncObj* pFuncObj = mnodeGetFunc(buf);
+    if (pFuncObj == NULL) {
+      mError("function %s does not exist", buf);
+      code = TSDB_CODE_MND_INVALID_FUNC;
+      goto _end;
+    }
+
+    SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) msg;
+
+    strcpy(pFuncInfo->name, buf);
+    pFuncInfo->len = htonl(pFuncObj->contLen);
+    memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
+
+    pFuncInfo->funcType = htonl(pFuncObj->funcType);
+    pFuncInfo->resType  = pFuncObj->resType;
+    pFuncInfo->resBytes = htons(pFuncObj->resBytes);
+    pFuncInfo->bufSize  = htonl(pFuncObj->bufSize);
+
+    msg += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
+  }
+
+  pMultiMeta->contLen  = (int32_t) (msg - (char*) pMultiMeta);
+  pMultiMeta->numOfUdf = htonl(pInfo->numOfUdfs);
+
   pMsg->rpcRsp.rsp = pMultiMeta;
   pMsg->rpcRsp.len = pMultiMeta->contLen;
   code = TSDB_CODE_SUCCESS;
@@ -3053,11 +3087,12 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
     goto _end;
   }
 
-  int32_t len = tsCompressString(pMultiMeta->meta, (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta), 1,
-      tmp + sizeof(SMultiTableMeta), (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta) + 2, ONE_STAGE_COMP, NULL, 0);
+  int32_t dataLen = (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta);
+  int32_t len = tsCompressString(pMultiMeta->meta, dataLen, 1, tmp + sizeof(SMultiTableMeta), (int32_t)dataLen + 2,
+                                 ONE_STAGE_COMP, NULL, 0);
 
   pMultiMeta->rawLen = pMultiMeta->contLen;
-  if (len == -1 || len + sizeof(SMultiTableMeta) >= pMultiMeta->contLen + 2) { // compress failed, do not compress this binary data
+  if (len == -1 || len >= dataLen + 2) { // compress failed, do not compress this binary data
     pMultiMeta->compressed = 0;
     memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta) + pMultiMeta->contLen);
   } else {

src/os/inc/os.h

@@ -37,6 +37,7 @@ extern "C" {
 #include "osSysinfo.h"
 #include "osTime.h"
 #include "osTimer.h"
+#include "osSystem.h"
 
 void osInit();
 

src/os/inc/osInc.h

@@ -76,7 +76,8 @@ extern "C" {
     #include <dispatch/dispatch.h>
     #include "osEok.h"
   #else
-    #include <argp.h> 
+    #include <argp.h>
+    #include <dlfcn.h>
     #include <endian.h>
     #include <linux/sysctl.h>
     #include <poll.h>

src/os/inc/osSystem.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_SYSTEM_H
+#define TDENGINE_OS_SYSTEM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void* taosLoadDll(const char *filename);
+void* taosLoadSym(void* handle, char* name);
+void taosCloseDll(void *handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/src/darwin/darwinSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void* taosLoadDll(const char *filename) {
+  return NULL;
+}
+
+void* taosLoadSym(void* handle, char* name) {
+  return NULL;
+}
+
+void taosCloseDll(void *handle) {
+}
+
+

src/os/src/linux/CMakeLists.txt

@@ -4,4 +4,4 @@ PROJECT(TDengine)
 AUX_SOURCE_DIRECTORY(. SRC)
 ADD_LIBRARY(oslinux ${SRC})
 
-TARGET_LINK_LIBRARIES(oslinux m rt z)
+TARGET_LINK_LIBRARIES(oslinux m rt z dl)
\ No newline at end of file

src/os/src/linux/osSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void* taosLoadDll(const char *filename) {
+  void *handle = dlopen (filename, RTLD_LAZY);  
+  if (!handle) {  
+    uError("load dll:%s failed, error:%s", filename, dlerror());  
+    return NULL;  
+  }
+
+  uDebug("dll %s loaded", filename);
+
+  return handle;
+}
+
+void* taosLoadSym(void* handle, char* name) {
+	void* sym = dlsym(handle, name);
+  char* error = NULL;
+  
+	if ((error = dlerror()) != NULL)  {  
+    uWarn("load sym:%s failed, error:%s", name, dlerror());  
+		return NULL;  
+	} 
+
+  uDebug("sym %s loaded", name)
+
+  return sym;
+}
+
+void taosCloseDll(void *handle) {
+  if (handle) {
+    dlclose(handle);
+  }
+}
+
+

src/os/src/windows/wSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void* taosLoadDll(const char *filename) {
+  return NULL;
+}
+
+void* taosLoadSym(void* handle, char* name) {
+  return NULL;
+}
+
+void taosCloseDll(void *handle) {
+}
+
+

src/plugins/http/inc/httpParser.h

@@ -17,7 +17,7 @@
 #define HTTP_PARSER_H
 #include "httpGzip.h"
 
-#define HTTP_MAX_URL 5  // http url stack size
+#define HTTP_MAX_URL 6  // http url stack size
 
 #define HTTP_CODE_CONTINUE                 100
 #define HTTP_CODE_SWITCHING_PROTOCOL       101

src/plugins/http/src/httpRestHandle.c

@@ -119,6 +119,90 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
   return true;
 }
 
+#define REST_FUNC_URL_POS   2
+#define REST_OUTP_URL_POS   3
+#define REST_AGGR_URL_POS   4
+#define REST_BUFF_URL_POS   5
+
+#define HTTP_FUNC_LEN  32
+#define HTTP_OUTP_LEN  16
+#define HTTP_AGGR_LEN  2
+#define HTTP_BUFF_LEN  32
+
+static int udfSaveFile(const char *fname, const char *content, long len) {
+  int fd = open(fname, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
+  if (fd < 0)
+    return -1;
+  if (taosWrite(fd, (void *)content, len) < 0)
+    return -1;
+
+  return 0;
+}
+
+static bool restProcessUdfRequest(HttpContext* pContext) {
+  HttpParser* pParser = pContext->parser;
+  if (pParser->path[REST_FUNC_URL_POS].pos >= HTTP_FUNC_LEN || pParser->path[REST_FUNC_URL_POS].pos <= 0) {
+    return false;
+  }
+
+  if (pParser->path[REST_OUTP_URL_POS].pos >= HTTP_OUTP_LEN || pParser->path[REST_OUTP_URL_POS].pos <= 0) {
+    return false;
+  }
+
+  if (pParser->path[REST_AGGR_URL_POS].pos >= HTTP_AGGR_LEN || pParser->path[REST_AGGR_URL_POS].pos <= 0) {
+    return false;
+  }
+
+  if (pParser->path[REST_BUFF_URL_POS].pos >= HTTP_BUFF_LEN || pParser->path[REST_BUFF_URL_POS].pos <= 0) {
+    return false;
+  }
+
+  char* sql = pContext->parser->body.str;
+  int len = pContext->parser->body.pos;
+  if (sql == NULL) {
+    httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
+    return false;
+  }
+
+  char udfDir[256] = {0};
+  char buf[64] = "udf-";
+  char funcName[64] = {0};
+  int aggr = 0;
+  char outputType[16] = {0};
+  char buffSize[32] = {0};
+
+  tstrncpy(funcName, pParser->path[REST_FUNC_URL_POS].str, HTTP_FUNC_LEN);
+  tstrncpy(buf + 4, funcName, HTTP_FUNC_LEN);
+
+  if (pParser->path[REST_AGGR_URL_POS].str[0] != '0') {
+    aggr = 1;
+  }
+
+  tstrncpy(outputType, pParser->path[REST_OUTP_URL_POS].str, HTTP_OUTP_LEN);
+  tstrncpy(buffSize, pParser->path[REST_BUFF_URL_POS].str, HTTP_BUFF_LEN);
+
+  taosGetTmpfilePath(funcName, udfDir);
+
+  udfSaveFile(udfDir, sql, len);
+
+  tfree(sql);
+  pContext->parser->body.str = malloc(1024);
+  sql = pContext->parser->body.str;
+  int sqlLen = sprintf(sql, "create %s function %s as \"%s\" outputtype %s bufsize %s",
+          aggr == 1  ? "aggregate" : " ", funcName, udfDir, outputType, buffSize);
+
+  pContext->parser->body.pos = sqlLen;
+  pContext->parser->body.size = sqlLen + 1;
+
+  HttpSqlCmd* cmd = &(pContext->singleCmd);
+  cmd->nativSql = sql;
+
+  pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
+  pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;
+
+  return true;
+}
+
 bool restProcessRequest(struct HttpContext* pContext) {
   if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
     restGetUserFromUrl(pContext);
@@ -138,6 +222,8 @@ bool restProcessRequest(struct HttpContext* pContext) {
     return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_UTC_STRING);
   } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
     return restProcessLoginRequest(pContext);
+  } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "udf")) {
+    return restProcessUdfRequest(pContext);
   } else {
   }
 

src/query/CMakeLists.txt

@@ -8,14 +8,14 @@ INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(query ${SRC})
 SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w)
-TARGET_LINK_LIBRARIES(query tsdb tutil)
+TARGET_LINK_LIBRARIES(query tsdb tutil lua)
 
 IF (TD_LINUX)
-  TARGET_LINK_LIBRARIES(query m rt)
+  TARGET_LINK_LIBRARIES(query m rt lua)
   ADD_SUBDIRECTORY(tests)
 ENDIF ()
 
 IF (TD_DARWIN)
-  TARGET_LINK_LIBRARIES(query m)
+  TARGET_LINK_LIBRARIES(query m lua)
   ADD_SUBDIRECTORY(tests)
 ENDIF ()

src/query/inc/qAggMain.h

@@ -27,6 +27,7 @@ extern "C" {
 #include "trpc.h"
 #include "tvariant.h"
 #include "tsdb.h"
+#include "qUdf.h"
 
 #define TSDB_FUNC_INVALID_ID  -1
 #define TSDB_FUNC_COUNT        0
@@ -201,10 +202,8 @@ typedef struct SAggFunctionInfo {
   int8_t   stableFuncId;  // transfer function for super table query
   uint16_t status;
 
-  bool (*init)(SQLFunctionCtx *pCtx);  // setup the execute environment
-
+  bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo);  // setup the execute environment
   void (*xFunction)(SQLFunctionCtx *pCtx);                     // blocks version function
-//  void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position);  // single-row function version, todo merge with blockwise function
 
   // finalizer must be called after all xFunction has been executed to generated final result.
   void (*xFinalize)(SQLFunctionCtx *pCtx);
@@ -216,7 +215,7 @@ typedef struct SAggFunctionInfo {
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
-                          int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable);
+                          int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo);
 int32_t isValidFunction(const char* name, int32_t len);
 
 #define IS_STREAM_QUERY_VALID(x)  (((x)&TSDB_FUNCSTATE_STREAM) != 0)
@@ -262,9 +261,9 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
 static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
   pResInfo->initialized = true;  // the this struct has been initialized flag
   
-  pResInfo->complete = false;
+  pResInfo->complete  = false;
   pResInfo->hasResult = false;
-  pResInfo->numOfRes = 0;
+  pResInfo->numOfRes  = 0;
   
   memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
 }

src/query/inc/qExecutor.h

@@ -29,6 +29,7 @@
 #include "tarray.h"
 #include "tlockfree.h"
 #include "tsdb.h"
+#include "qUdf.h"
 
 struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
@@ -257,6 +258,7 @@ typedef struct SQueryAttr {
   SMemRef          memRef;
   STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
   int32_t          vgId;
+  SArray          *pUdfInfo;             // no need to free
 } SQueryAttr;
 
 typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
@@ -297,6 +299,7 @@ typedef struct SQueryRuntimeEnv {
   STableQueryInfo      *current;
   SRspResultInfo        resultInfo;
   SHashObj             *pTableRetrieveTsMap;
+  SUdfInfo             *pUdfInfo;
 } SQueryRuntimeEnv;
 
 enum {
@@ -392,6 +395,7 @@ typedef struct SQueryParam {
   SGroupbyExpr *pGroupbyExpr;
   int32_t          tableScanOperator;
   SArray          *pOperator;
+  SUdfInfo        *pUdfInfo;
 } SQueryParam;
 
 typedef struct STableScanInfo {
@@ -531,6 +535,7 @@ typedef struct SMultiwayMergeInfo {
 
   bool                 hasPrev;
   bool                 groupMix;
+  SArray              *udfInfo;
 } SMultiwayMergeInfo;
 
 void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
@@ -553,8 +558,8 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat
 SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
 SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
                                               int32_t numOfRows, void* merger, bool groupMix);
+SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo);
 SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
-SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param);
 SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger);
 SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr,
                                         int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
@@ -578,18 +583,19 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo
 int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput);
 void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
 void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows);
+void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity);
 
 void freeParam(SQueryParam *param);
 int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param);
 int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
-                        SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg);
+                        SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo);
 
 int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
-                                           SSqlExpr **pExpr, SExprInfo *prevExpr);
+                                           SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
 
 SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
 SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
-                        SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId);
+                        SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
 
 int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
                   int32_t prevResultLen, void* merger);
@@ -608,6 +614,8 @@ bool doBuildResCheck(SQInfo* pQInfo);
 void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status);
 
 bool onlyQueryTags(SQueryAttr* pQueryAttr);
+void destroyUdfInfo(SUdfInfo* pUdfInfo);
+
 bool isValidQInfo(void *param);
 
 int32_t doDumpQueryResult(SQInfo *pQInfo, char *data);
@@ -625,4 +633,6 @@ void freeQueryAttr(SQueryAttr *pQuery);
 
 int32_t getMaximumIdleDurationSec();
 
+void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
+
 #endif  // TDENGINE_QEXECUTOR_H

src/query/inc/qScript.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_QSCRIPT_H
+#define TDENGINE_QSCRIPT_H
+
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+
+#include "tutil.h"
+#include "hash.h"
+#include "tlist.h"
+#include "qUdf.h"
+
+#define MAX_FUNC_NAME 64
+
+#define USER_FUNC_NAME "funcName" 
+#define USER_FUNC_NAME_LIMIT 48
+
+enum ScriptState {
+  SCRIPT_STATE_INIT,
+  SCRIPT_STATE_ADD,
+  SCRIPT_STATE_MERGE,
+  SCRIPT_STATE_FINALIZE 
+};
+
+
+typedef struct {
+  SHashObj  *funcId;    //func already registed in lua_env, may be no use
+  lua_State *lua_state; // lua env 
+} ScriptEnv;   
+
+typedef struct ScriptCtx {
+  char        funcName[USER_FUNC_NAME_LIMIT]; 
+  int8_t      state; 
+  ScriptEnv  *pEnv;
+  int8_t      isAgg; // agg function or not
+  
+  // init value of udf script
+  int8_t      resType;
+  int16_t     resBytes; 
+
+  int32_t     numOfOutput; 
+  int32_t     offset;
+  
+} ScriptCtx;
+
+int taosLoadScriptInit(void *pInit);
+void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows, 
+    int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
+void taosLoadScriptFinalize(void *pInit, int64_t key, char *pOutput, int32_t *output);
+void taosLoadScriptMerge(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
+void taosLoadScriptDestroy(void *pInit);
+
+typedef struct {
+  SList     *scriptEnvs; //  
+  int32_t   mSize;  // pool limit
+  int32_t   cSize;  // current available size
+  pthread_mutex_t mutex;
+} ScriptEnvPool;
+
+ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes);
+void       destroyScriptCtx(void *pScriptCtx);
+
+int32_t scriptEnvPoolInit();
+void    scriptEnvPoolCleanup();
+bool    isValidScript(char *script, int32_t len);
+
+#endif //TDENGINE_QSCRIPT_H 

src/query/inc/qSqlparser.h

@@ -182,6 +182,15 @@ typedef struct SCreateDbInfo {
   int16_t            partitions;
 } SCreateDbInfo;
 
+typedef struct SCreateFuncInfo {
+  SStrToken name;
+  SStrToken path;
+  int32_t   type;
+  int32_t   bufSize;
+  TAOS_FIELD output;
+} SCreateFuncInfo;
+
+
 typedef struct SCreateAcctInfo {
   int32_t            maxUsers;
   int32_t            maxDbs;
@@ -214,10 +223,11 @@ typedef struct SMiscInfo {
   int16_t            tableType;
   SUserInfo          user;
   union {
-    SCreateDbInfo    dbOpt;
-    SCreateAcctInfo  acctOpt;
-    SShowInfo        showOpt;
-    SStrToken        id;
+    SCreateDbInfo   dbOpt;
+    SCreateAcctInfo acctOpt;
+    SCreateFuncInfo funcOpt;
+    SShowInfo       showOpt;
+    SStrToken       id;
   };
 } SMiscInfo;
 
@@ -226,6 +236,7 @@ typedef struct SSqlInfo {
   bool               valid;
   SArray            *list;    // todo refactor
   char               msg[256];
+  SArray            *funcs;
   union {
     SCreateTableSql *pCreateTableInfo;
     SAlterTableInfo *pAlterInfo;
@@ -271,6 +282,7 @@ SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrT
 // sql expr leaf node
 tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType);
 tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType);
+SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken);
 
 tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
 tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);

src/query/inc/qTableMeta.h

@@ -125,8 +125,12 @@ typedef struct SQueryInfo {
   int32_t          round;         // 0/1/....
   int32_t          bufLen;
   char*            buf;
-  struct SQInfo*          pQInfo;      // global merge operator
-  struct SQueryAttr*      pQueryAttr;     // query object
+
+  bool               udfCopy;
+  SArray            *pUdfInfo;
+
+  struct SQInfo     *pQInfo;      // global merge operator
+  struct SQueryAttr *pQueryAttr;     // query object
 
   struct SQueryInfo *sibling;     // sibling
   SArray            *pUpstream;   // SArray<struct SQueryInfo>
@@ -141,6 +145,7 @@ typedef struct SQueryInfo {
   bool               onlyTagQuery;
   bool               orderProjectQuery;
   bool               stateWindow;
+  bool               globalMerge;
 } SQueryInfo;
 
 /**

src/query/inc/qUdf.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_QUDF_H
+#define TDENGINE_QUDF_H
+
+enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM };
+
+
+
+typedef struct SUdfInit{
+ int32_t maybe_null;       /* 1 if function can return NULL */
+ uint32_t decimals;     /* for real functions */
+ uint64_t length;       /* For string functions */
+ char  *ptr;            /* free pointer for function data */
+ int32_t const_item;       /* 0 if result is independent of arguments */
+
+ // script like lua/javascript
+ void* script_ctx;
+ void (*destroyCtxFunc)(void *script_ctx);
+} SUdfInit;
+
+
+typedef struct SUdfInfo {
+  int32_t functionId;  // system assigned function id
+  int32_t funcType;    // scalar function or aggregate function
+  int8_t  resType;     // result type
+  int16_t resBytes;    // result byte
+  int32_t contLen;     // content length
+  int32_t bufSize;     //interbuf size
+  char   *name;        // function name
+  void   *handle;      // handle loaded in mem
+  void   *funcs[TSDB_UDF_FUNC_MAX_NUM];     // function ptr
+
+  // for script like lua/javascript only
+  int    isScript;
+  void   *pScriptCtx;
+
+  SUdfInit init;
+  char *content;
+  char *path;
+} SUdfInfo;
+
+//script 
+
+typedef int32_t (*scriptInitFunc)(void *pCtx);
+typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
+                                 int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes);
+typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
+typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
+typedef void (*scriptDestroyFunc)(void* pCtx);
+
+// dynamic lib
+typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf,
+  char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf);
+typedef int32_t (*udfInitFunc)(SUdfInit* data);
+typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf);
+typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
+typedef void (*udfDestroyFunc)(SUdfInit* buf);
+
+
+#endif  // TDENGINE_QUDF_H

src/query/inc/qUtil.h

@@ -104,4 +104,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 
 int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset);
 
+int32_t initUdfInfo(SUdfInfo* pUdfInfo);
+
 #endif  // TDENGINE_QUERYUTIL_H

src/query/inc/sql.y

@@ -65,6 +65,7 @@ program ::= cmd.    {}
 //////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
 cmd ::= SHOW DATABASES.  { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
 cmd ::= SHOW TOPICS.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
+cmd ::= SHOW FUNCTIONS.  { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
 cmd ::= SHOW MNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
 cmd ::= SHOW DNODES.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
 cmd ::= SHOW ACCOUNTS.   { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
@@ -147,6 +148,7 @@ cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z).   {
 
 cmd ::= DROP DATABASE ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
 cmd ::= DROP TOPIC ifexists(Y) ids(X).    { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
+cmd ::= DROP FUNCTION ids(X).    { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); }
 
 cmd ::= DROP DNODE ids(X).       { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X);    }
 cmd ::= DROP USER ids(X).        { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X);     }
@@ -200,8 +202,13 @@ cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
                                 { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
 cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
 cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
+cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B).   { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);}
+cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B).   { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);}
 cmd ::= CREATE USER ids(X) PASS ids(Y).     { setCreateUserSql(pInfo, &X, &Y);}
 
+bufsize(Y) ::= .                                { Y.n = 0;   }
+bufsize(Y) ::= BUFSIZE INTEGER(X).              { Y = X;     }
+
 pps(Y) ::= .                                { Y.n = 0;   }
 pps(Y) ::= PPS INTEGER(X).                  { Y = X;     }
 
@@ -704,10 +711,10 @@ expr(A) ::= BOOL(X).             { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
 expr(A) ::= NULL(X).             { A = tSqlExprCreateIdValue(&X, TK_NULL);}
 
 // ordinary functions: min(x), max(x), top(k, 20)
-expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
+expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
 
 // for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
-expr(A) ::= ID(X) LP STAR RP(Y).     { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
+expr(A) ::= ID(X) LP STAR RP(Y).     { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
 
 // is (not) null expression
 expr(A) ::= expr(X) IS NULL.           {A = tSqlExprCreate(X, NULL, TK_ISNULL);}

src/query/src/qAggMain.c

@@ -27,6 +27,7 @@
 #include "qPercentile.h"
 #include "qTsbuf.h"
 #include "queryLog.h"
+#include "qUdf.h"
 
 #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
 #define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
@@ -169,12 +170,13 @@ typedef struct SDerivInfo {
 } SDerivInfo;
 
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
-                          int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
+                          int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) {
   if (!isValidDataType(dataType)) {
     qError("Illegal data type %d or data type length %d", dataType, dataBytes);
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
-  
+
+
   if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY ||
       functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ ||
       functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
@@ -234,6 +236,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   }
 
   if (isSuperTable) {
+    if (functionId < 0) {
+      if (pUdfInfo->bufSize > 0) {
+        *type = TSDB_DATA_TYPE_BINARY;
+        *bytes = pUdfInfo->bufSize;
+        *interBytes = *bytes;
+      } else {
+        *type = pUdfInfo->resType;
+        *bytes = pUdfInfo->resBytes;
+        *interBytes = *bytes;
+      }
+
+      return TSDB_CODE_SUCCESS;
+    }
+
     if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) {
       *type = TSDB_DATA_TYPE_BINARY;
       *bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE);
@@ -288,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
       return TSDB_CODE_SUCCESS;
     }
   }
-  
+
   if (functionId == TSDB_FUNC_SUM) {
     if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
       *type = TSDB_DATA_TYPE_BIGINT;
@@ -313,7 +329,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
     *interBytes = sizeof(STwaInfo);
     return TSDB_CODE_SUCCESS;
   }
-  
+
+  if (functionId < 0) {
+    *type = pUdfInfo->resType;
+    *bytes = pUdfInfo->resBytes;
+
+    if (pUdfInfo->bufSize > 0) {
+      *interBytes = pUdfInfo->bufSize;
+    } else {
+      *interBytes = *bytes;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (functionId == TSDB_FUNC_AVG) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
@@ -390,14 +419,13 @@ int32_t isValidFunction(const char* name, int32_t len) {
   return -1;
 }
 
-static bool function_setup(SQLFunctionCtx *pCtx) {
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  if (pResInfo->initialized) {
+static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (pResultInfo->initialized) {
     return false;
   }
   
   memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes);
-  initResultInfo(pResInfo, pCtx->interBufBytes);
+  initResultInfo(pResultInfo, pCtx->interBufBytes);
   return true;
 }
 
@@ -985,8 +1013,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
   }
 }
 
-static bool min_func_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
     return false;  // not initialized since it has been initialized
   }
   
@@ -1030,8 +1058,8 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) {
   return true;
 }
 
-static bool max_func_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
     return false;  // not initialized since it has been initialized
   }
   
@@ -1435,8 +1463,8 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 //////////////////////////////////////////////////////////////////////////////////////
-static bool first_last_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
   
@@ -1606,7 +1634,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
       break;
     }
   }
-  
+
   SET_VAL(pCtx, notNullElems, 1);
 }
 
@@ -2007,7 +2035,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) {
   char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo);
   pTopBotInfo->res = (tValuePair**) tmp;
   tmp += POINTER_BYTES * pCtx->param[0].i64;
-  
+
   size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen;
 
   for (int32_t i = 0; i < pCtx->param[0].i64; ++i) {
@@ -2073,14 +2101,13 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
   }
 }
 
-static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
   
   STopBotInfo *pInfo = getTopBotOutputInfo(pCtx);
   buildTopBotStruct(pInfo, pCtx);
-  
   return true;
 }
 
@@ -2229,14 +2256,13 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
-static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
     return false;
   }
 
   // in the first round, get the min-max value of all involved data
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
   SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
   SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
   pInfo->numOfElems = 0;
@@ -2367,8 +2393,8 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
   return pInfo;
 }
 
-static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
     return false;
   }
   
@@ -2477,12 +2503,11 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
 }
 
 /////////////////////////////////////////////////////////////////////////////////
-static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
-  
-  SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
+
   SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   
   // 2*3 matrix
@@ -2706,8 +2731,8 @@ enum {
   INITIAL_VALUE_NOT_ASSIGNED = 0,
 };
 
-static bool diff_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
   
@@ -2716,14 +2741,13 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) {
   return false;
 }
 
-static bool deriv_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
     return false;
   }
 
   // diff function require the value is set to -1
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo);
 
   pDerivInfo->ignoreNegative = pCtx->param[1].i64;
   pDerivInfo->prevTs   = -1;
@@ -3130,12 +3154,12 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) {
   }
 
 /////////////////////////////////////////////////////////////////////////////////
-static bool spread_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
   
-  SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   
   // this is the server-side setup function in client-side, the secondary merge do not need this procedure
   if (pCtx->currentStage == MERGE_STAGE) {
@@ -3289,12 +3313,10 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
  * param[2]: end time
  * @param pCtx
  */
-static bool twa_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
-  
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
 
   STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   pInfo->p.key    = INT64_MIN;
@@ -3726,14 +3748,12 @@ static void interp_function(SQLFunctionCtx *pCtx) {
   }
 }
 
-static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;  // not initialized since it has been initialized
   }
-  
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
   STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
-  
   pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
   pInfo->pTSBuf->tsOrder = pCtx->order;
   return true;
@@ -3816,13 +3836,11 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) {
   return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0;
 }
 
-static bool rate_function_setup(SQLFunctionCtx *pCtx) {
-  if (!function_setup(pCtx)) {
+static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
     return false;
   }
   
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-
   SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   pInfo->correctionValue = 0;
   pInfo->firstKey    = INT64_MIN;
@@ -3920,7 +3938,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
   int32_t    notNullElems = 0;
   SRateInfo *pRateInfo    = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
   TSKEY     *primaryKey   = GET_TS_LIST(pCtx);
-  
+
   for (int32_t i = pCtx->size - 1; i >= 0; --i) {
     char *pData = GET_INPUT_DATA(pCtx, i);
     if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {

src/query/src/qScript.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "qScript.h"
+#include "ttype.h"
+#include "tstrbuild.h"
+#include "queryLog.h"
+#include "ttokendef.h"
+
+static ScriptEnvPool *pool = NULL;  
+
+static ScriptEnv* getScriptEnvFromPool();
+static void addScriptEnvToPool(ScriptEnv *pEnv);
+
+static lua_State* createLuaEnv();
+static void destroyLuaEnv(lua_State *state);
+
+static void destroyScriptEnv(ScriptEnv *pEnv);
+
+static void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
+static void taosValueToLuaType(lua_State *lua, int32_t type, char *val);
+
+static bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len);
+
+static int userlib_exampleFunc(lua_State *lua) {
+  double op1 = luaL_checknumber(lua,1);
+  double op2 = luaL_checknumber(lua,2);
+  lua_pushnumber(lua, op1 * op2); 
+  return 1; 
+}  
+void luaRegisterLibFunc(lua_State *lua) {
+  lua_register(lua, "exampleFunc", userlib_exampleFunc);
+}
+
+void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) {
+  lua_pushcfunction(lua, luafunc);
+  lua_pushstring(lua, libname);
+  lua_call(lua, 1, 0);
+}
+
+LUALIB_API int (luaopen_cjson) (lua_State *L);
+LUALIB_API int (luaopen_struct) (lua_State *L);
+LUALIB_API int (luaopen_cmsgpack) (lua_State *L);
+LUALIB_API int (luaopen_bit) (lua_State *L);
+
+
+static void luaLoadLibraries(lua_State *lua) {
+  luaLoadLib(lua, "", luaopen_base);
+  luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table);
+  luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string);
+  luaLoadLib(lua, LUA_MATHLIBNAME, luaopen_math);
+  luaLoadLib(lua, LUA_DBLIBNAME, luaopen_debug);
+}
+static void luaRemoveUnsupportedFunctions(lua_State *lua) {
+  lua_pushnil(lua);
+  lua_setglobal(lua,"loadfile");
+  lua_pushnil(lua);
+  lua_setglobal(lua,"dofile");
+}
+void taosValueToLuaType(lua_State *lua, int32_t type, char *val) {
+  //TODO(dengyihao): handle more data type
+  if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
+    int64_t v;
+    GET_TYPED_DATA(v, int64_t, type, val);
+    lua_pushnumber(lua, (lua_Number)v);  
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+    uint64_t v;
+    GET_TYPED_DATA(v, uint64_t, type, val);
+    lua_pushnumber(lua,(lua_Number)v);  
+  } else if (IS_FLOAT_TYPE(type)) {
+    double v; 
+    GET_TYPED_DATA(v, double, type, val);
+    lua_pushnumber(lua,v); 
+  } else if (type == TSDB_DATA_TYPE_BINARY) {
+    lua_pushlstring(lua, val, varDataLen(val));       
+  } else if (type == TSDB_DATA_TYPE_NCHAR) {
+  } 
+} 
+int taosLoadScriptInit(void* pInit) {
+  ScriptCtx *pCtx = pInit;   
+  char funcName[MAX_FUNC_NAME] = {0};
+  sprintf(funcName, "%s_init", pCtx->funcName);
+
+  lua_State* lua = pCtx->pEnv->lua_state;   
+  lua_getglobal(lua, funcName);
+  if (lua_pcall(lua, 0, -1, 0)) {
+    lua_pop(lua, -1);
+  }
+  lua_setglobal(lua, "global"); 
+  return 0;
+}
+void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows, 
+    int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes) { 
+  ScriptCtx* pCtx = pInit;
+  char funcName[MAX_FUNC_NAME] = {0};
+  sprintf(funcName, "%s_add", pCtx->funcName);
+
+  lua_State* lua  = pCtx->pEnv->lua_state;   
+  lua_getglobal(lua, funcName);
+
+  // first param of script;
+  lua_newtable(lua); 
+  int32_t offset = 0;
+  for (int32_t i = 0; i < numOfRows; i++) {
+    taosValueToLuaType(lua, iType, pInput + offset); 
+    lua_rawseti(lua, -2, i+1);
+    offset += iBytes;
+  } 
+  int isGlobalState = false; 
+  lua_getglobal(lua, "global"); 
+  if (lua_istable(lua, -1)) {
+    isGlobalState = true; 
+  }
+  lua_pushnumber(lua, (lua_Number)key);
+  // do call lua script 
+  if (lua_pcall(lua, 3, 1, 0) != 0) {
+    qError("SCRIPT ERROR: %s", lua_tostring(lua, -1)); 
+    lua_pop(lua, -1);
+    return;
+  }
+  int tNumOfOutput = 0; 
+  if (isGlobalState == false) {
+    luaValueToTaosType(lua, pOutput, &tNumOfOutput, oType, oBytes);
+  } else {
+    lua_setglobal(lua, "global");
+  }
+  *numOfOutput = tNumOfOutput;
+}
+
+void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutput, int32_t* numOfOutput) {
+  ScriptCtx *pCtx = pInit;
+  char funcName[MAX_FUNC_NAME] = {0};
+  sprintf(funcName, "%s_merge", pCtx->funcName);
+
+  lua_State* lua  = pCtx->pEnv->lua_state;   
+  lua_getglobal(lua, funcName);
+  if (!lua_isfunction(lua, -1)) {
+    qError("SCRIPT ERROR: %s", lua_tostring(lua, -1)); 
+    return;
+  }
+
+  lua_getglobal(lua, "global"); 
+  if (lua_pcall(lua, 1, 1, 0) != 0) {
+    qError("SCRIPT ERROR: %s", lua_tostring(lua, -1)); 
+    lua_pop(lua, -1);
+    return;
+  }
+  int tNumOfOutput = 0; 
+  luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
+  *numOfOutput = tNumOfOutput;
+}
+
+//do not support agg now
+void taosLoadScriptFinalize(void *pInit,int64_t key, char *pOutput, int32_t* numOfOutput) {
+  ScriptCtx *pCtx = pInit;
+  char funcName[MAX_FUNC_NAME] = {0};
+  sprintf(funcName, "%s_finalize", pCtx->funcName);
+  
+  lua_State* lua  = pCtx->pEnv->lua_state;   
+  lua_getglobal(lua, funcName);
+  if (!lua_isfunction(lua, -1)) {
+    qError("SCRIPT ERROR: %s", lua_tostring(lua, -1)); 
+    return;
+  }
+
+  lua_getglobal(lua, "global"); 
+
+  lua_pushnumber(lua, (lua_Number)key);
+  if (lua_pcall(lua, 2, 2, 0) != 0) {
+    qError("SCRIPT ERROR: %s", lua_tostring(lua, -1)); 
+    lua_pop(lua, -1);
+    return;
+  }
+  lua_setglobal(lua, "global"); 
+  int tNumOfOutput = 0; 
+  luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
+  *numOfOutput = tNumOfOutput;
+}
+
+void taosLoadScriptDestroy(void *pInit) {
+  destroyScriptCtx(pInit);
+}
+
+ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) {
+  ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx)); 
+  pCtx->state = SCRIPT_STATE_INIT; 
+  pCtx->pEnv  = getScriptEnvFromPool();  //  
+  pCtx->resType  = resType;
+  pCtx->resBytes = resBytes; 
+  
+  if (pCtx->pEnv == NULL) {
+    destroyScriptCtx(pCtx);
+    return NULL;
+  } 
+
+  lua_State *lua = pCtx->pEnv->lua_state;
+  if (luaL_dostring(lua, script)) {
+    lua_pop(lua, 1);
+    qError("dynamic load script failed");
+    destroyScriptCtx(pCtx);
+    return NULL;
+  } 
+  lua_getglobal(lua, USER_FUNC_NAME);
+  const char *name = lua_tostring(lua, -1); 
+  if (name == NULL) {
+    lua_pop(lua, 1);
+    qError("SCRIPT ERROR: invalid script");
+    destroyScriptCtx(pCtx);
+    return NULL; 
+  }
+  memcpy(pCtx->funcName, name, strlen(name));
+  lua_pop(lua, 1);
+  
+  return pCtx;
+}
+void destroyScriptCtx(void *pCtx) {
+  if (pCtx == NULL) return;
+  addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv);
+  free(pCtx);
+}
+
+void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
+  int t = lua_type(lua,-1); 
+  int32_t sz = 0;
+  switch (t) {
+    case LUA_TSTRING:
+      //TODO(yihaodeng): handle str type
+      {
+        const char *v = lua_tostring(lua, -1);
+        memcpy(interBuf, v, strlen(v));
+        sz = 1; 
+      }
+      break;
+    case LUA_TBOOLEAN: 
+      {
+        double v = lua_tonumber(lua, -1);
+        memcpy(interBuf, (char *)&v, oBytes);
+        sz = 1;
+      }
+      break;
+    case LUA_TNUMBER:
+      {
+        if (oType == TSDB_DATA_TYPE_FLOAT) {
+          float v = (float)lua_tonumber(lua, -1);
+          memcpy(interBuf, (char *)&v, oBytes);
+          sz = 1;
+        } else if (oType == TSDB_DATA_TYPE_DOUBLE) {
+          double v = (double)lua_tonumber(lua, -1);
+          memcpy(interBuf, (char *)&v, oBytes);
+          sz = 1;
+        } else if (oType == TSDB_DATA_TYPE_BIGINT) {
+          int64_t v = (int64_t)lua_tonumber(lua, -1);
+          memcpy(interBuf, (char *)&v, oBytes);
+          sz = 1;
+        } else if (oType <= TSDB_DATA_TYPE_INT) {
+          int32_t v = (int32_t)lua_tonumber(lua, -1);
+          memcpy(interBuf, (char *)&v, oBytes);
+          sz = 1;
+        }
+      }
+      break;
+    case LUA_TTABLE: 
+      {
+        lua_pushnil(lua);
+        int32_t offset = 0;
+        while(lua_next(lua, -2)) {
+          int32_t v = (int32_t)lua_tonumber(lua, -1); 
+          memcpy(interBuf + offset, (char *)&v, oBytes);
+          offset += oBytes;
+          lua_pop(lua, 1);
+          sz += 1;
+        }
+      }
+      break;
+    default:
+      setNull(interBuf, oType, oBytes);
+      sz = 1;
+      break; 
+  }
+  lua_pop(lua,1); // pop ret value from script  
+  *numOfOutput = sz; 
+}
+
+
+/*
+*Initialize the scripting environment.
+*/  
+lua_State* createLuaEnv() {
+  lua_State *lua = lua_open(); 
+  luaLoadLibraries(lua);
+  luaRemoveUnsupportedFunctions(lua);
+
+  // register func in external lib 
+  luaRegisterLibFunc(lua); 
+
+  {
+    char *errh_func = "local dbg = debug\n"
+                      "function __taos__err__handler(err)\n"
+                      "  local i = dbg.getinfo(2,'nSl')\n"
+                      "  if i and i.what == 'C' then\n"
+                      "    i = dbg.getinfo(3,'nSl')\n"
+                      "  end\n"
+                      "  if i then\n"
+                      "    return i.source .. ':' .. i.currentline .. ': ' .. err\n"
+                      "  else\n"
+                      "    return err\n"
+                      "  end\n"
+                      "end\n";
+    luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def");
+    lua_pcall(lua,0,0,0);
+  }
+    
+  return lua;
+}
+
+void destroyLuaEnv(lua_State *lua) {
+  lua_close(lua); 
+}
+
+int32_t scriptEnvPoolInit() {
+  const int size = 10; // configure or not 
+  pool = malloc(sizeof(ScriptEnvPool));  
+  pthread_mutex_init(&pool->mutex, NULL);
+
+  pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *));
+  for (int i = 0; i < size; i++) {
+    ScriptEnv *env = malloc(sizeof(ScriptEnv));
+    env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);;  
+    env->lua_state = createLuaEnv(); 
+    tdListAppend(pool->scriptEnvs, (void *)(&env));  
+  }
+  pool->mSize = size;
+  pool->cSize = size;
+  return 0;  
+}
+
+void scriptEnvPoolCleanup() {
+  if (pool == NULL) return;
+  
+  SListNode *pNode = NULL; 
+  while ((pNode = tdListPopHead(pool->scriptEnvs)) != NULL) {
+    ScriptEnv *pEnv = NULL;
+    tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
+    destroyScriptEnv(pEnv);
+    listNodeFree(pNode); 
+  }  
+  tdListFree(pool->scriptEnvs);
+  pthread_mutex_destroy(&pool->mutex);
+  free(pool);
+}
+
+void destroyScriptEnv(ScriptEnv *pEnv) {
+  destroyLuaEnv(pEnv->lua_state);       
+  taosHashCleanup(pEnv->funcId); 
+  free(pEnv);
+} 
+
+ScriptEnv* getScriptEnvFromPool() {
+  ScriptEnv *pEnv = NULL;
+
+  pthread_mutex_lock(&pool->mutex); 
+  if (pool->cSize <= 0) {
+    pthread_mutex_unlock(&pool->mutex); 
+    return NULL;
+  }  
+  SListNode *pNode = tdListPopHead(pool->scriptEnvs);
+  tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
+  listNodeFree(pNode); 
+  
+  pool->cSize--;
+  pthread_mutex_unlock(&pool->mutex); 
+  return pEnv; 
+}
+
+void addScriptEnvToPool(ScriptEnv *pEnv) {
+  if (pEnv == NULL) {
+    return;
+  }
+  pthread_mutex_lock(&pool->mutex); 
+  lua_settop(pEnv->lua_state, 0);
+  tdListAppend(pool->scriptEnvs, (void *)(&pEnv));  
+  pool->cSize++;
+  pthread_mutex_unlock(&pool->mutex); 
+}
+
+bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) {
+  bool ret = true;
+  char funcName[MAX_FUNC_NAME]; 
+  memcpy(funcName, funcPrefix, len); 
+
+  const char *base[] = {"_init", "_add"};
+  for (int i = 0; (i < sizeof(base)/sizeof(base[0])) && (ret == true); i++) {
+    memcpy(funcName + len, base[i], strlen(base[i])); 
+    memset(funcName + len + strlen(base[i]), 0, MAX_FUNC_NAME - len - strlen(base[i]));
+    lua_getglobal(lua, funcName);   
+    ret = lua_isfunction(lua, -1); // exsit function or not 
+    lua_pop(lua, 1);
+  }
+  return ret;
+} 
+
+bool isValidScript(char *script, int32_t len) {
+  ScriptEnv *pEnv = getScriptEnvFromPool();  //  
+  if (pEnv == NULL) {
+    return false;
+  }  
+  lua_State *lua = pEnv->lua_state; 
+  if (len < strlen(script)) {
+    script[len] = 0; 
+  }  
+  if (luaL_dostring(lua, script)) {
+    lua_pop(lua, 1);
+    addScriptEnvToPool(pEnv); 
+    qError("error at %s and %d", script, (int)(strlen(script)));
+    return false;
+  }
+  lua_getglobal(lua, USER_FUNC_NAME);
+  const char *name = lua_tostring(lua, -1);
+  if (name == NULL || strlen(name) >= USER_FUNC_NAME_LIMIT) {
+    lua_pop(lua, 1);
+    addScriptEnvToPool(pEnv); 
+    qError("error at %s name: %s, len = %d", script, name, (int)(strlen(name)));
+    return false;
+  } 
+  bool ret = hasBaseFuncDefinedInScript(lua, name, (int32_t)strlen(name));
+  lua_pop(lua, 1); // pop  
+  addScriptEnvToPool(pEnv); 
+  return ret;
+}
+

src/query/src/qSqlParser.c

@@ -28,6 +28,7 @@ SSqlInfo qSqlParse(const char *pStr) {
 
   SSqlInfo sqlInfo = {0};
   sqlInfo.valid = true;
+  sqlInfo.funcs = taosArrayInit(4, sizeof(SStrToken));
 
   int32_t i = 0;
   while (1) {
@@ -120,6 +121,19 @@ void tSqlExprListDestroy(SArray *pList) {
   taosArrayDestroyEx(pList, freeExprElem);
 }
 
+
+SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) {
+  if (pList == NULL) {
+    pList = taosArrayInit(4, sizeof(tVariantListItem));
+  }
+
+  if (pToken) {
+    taosArrayPush(pList, pToken);
+  }
+
+  return pList;
+}
+
 tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) {
   tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr));
 
@@ -922,8 +936,8 @@ void* destroyCreateTableSql(SCreateTableSql* pCreate) {
 }
 
 void SqlInfoDestroy(SSqlInfo *pInfo) {
-  if (pInfo == NULL) return;
-
+  if (pInfo == NULL) return;;
+  taosArrayDestroy(pInfo->funcs); 
   if (pInfo->type == TSDB_SQL_SELECT) {
     destroyAllSqlNode(pInfo->list);
   } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) {
@@ -1018,6 +1032,18 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrTo
   pInfo->pMiscInfo->tableType = tableType;
 }
 
+void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken) {
+  pInfo->type = type;
+
+  if (pInfo->pMiscInfo == NULL) {
+    pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo));
+    pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken));
+  }
+
+  taosArrayPush(pInfo->pMiscInfo->a, pToken);
+}
+
+
 void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) {
   if (pInfo->pMiscInfo == NULL) {
     pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
@@ -1052,6 +1078,24 @@ void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDb
   pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS.   {X.n = 1;}
 }
 
+void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPath, TAOS_FIELD *output, SStrToken* bufSize, int32_t funcType) {
+  pInfo->type = type;
+  if (pInfo->pMiscInfo == NULL) {
+    pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
+  }
+
+  pInfo->pMiscInfo->funcOpt.name = *pName;
+  pInfo->pMiscInfo->funcOpt.path = *pPath;
+  pInfo->pMiscInfo->funcOpt.output = *output;
+  pInfo->pMiscInfo->funcOpt.type = funcType;
+  if (bufSize->n > 0) {
+    pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10);
+  } else {
+    pInfo->pMiscInfo->funcOpt.bufSize = 0;  
+  }
+}
+
+
 void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) {
   pInfo->type = type;
   if (pInfo->pMiscInfo == NULL) {

src/query/src/queryMain.c

@@ -68,7 +68,7 @@ void freeParam(SQueryParam *param) {
   tfree(param->prevResult);
 }
 
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t *qId) {
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) {
   assert(pQueryMsg != NULL && tsdb != NULL);
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -93,12 +93,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
 
   SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
   if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, &param.pExprs, param.pExpr, param.pTagColumnInfo,
-                              pQueryMsg->queryType, pQueryMsg)) != TSDB_CODE_SUCCESS) {
+                              pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
     goto _over;
   }
 
   if (param.pSecExpr != NULL) {
-    if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs)) != TSDB_CODE_SUCCESS) {
+    if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }
   }
@@ -162,18 +162,14 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
 
   assert(pQueryMsg->stableQuery == isSTableQuery);
   (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
-                              param.pTagColumnInfo, vgId, param.sql, *qId);
+                              param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo);
 
   param.sql    = NULL;
   param.pExprs = NULL;
   param.pSecExprs = NULL;
   param.pGroupbyExpr = NULL;
   param.pTagColumnInfo = NULL;
-
-  if ((*pQInfo) == NULL) {
-    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
-    goto _over;
-  }
+  param.pUdfInfo = NULL;
 
   code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
 
@@ -182,6 +178,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
     taosArrayDestroy(param.pGroupbyExpr->columnInfo);
   }
 
+  destroyUdfInfo(param.pUdfInfo);
+
   taosArrayDestroy(param.pTableIdList);
   param.pTableIdList = NULL;
 

src/util/src/terror.c

@@ -109,11 +109,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR,         "Syntax error in SQL")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED,          "Database not specified or available")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME,       "Table does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT,         "SQL statement too long, check maxSQLLength config")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY,               "File is empty")
 
 // mnode
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED,        "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS,       "Message is progressing")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED,  "Messag need to be reprocessed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED,  "Message need to be reprocessed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS,                "Insufficient privilege for operation")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR,                "Unexpected generic error in mnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION,       "Invalid message connection")
@@ -183,6 +184,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          "Field does not exist"
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME,      "Super table does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message")
 
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME,        "Invalid func name")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN,         "Invalid func length")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE,        "Invalid func code")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST,       "Func already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC,             "Invalid func")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE,     "Invalid func bufSize")
+
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED,          "Database not specified or available")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST,         "Database already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION,        "Invalid database options")
@@ -269,6 +277,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC,                  "Multiple retrieval of
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW,      "Too many time window in query")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,        "Query buffer limit has reached")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN,              "File inconsistance in replica")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR,                "System error")
 
 
 // grant

src/util/src/tskiplist.c

@@ -546,7 +546,6 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
   const uint32_t factor = 4;
 
   int32_t n = 1;
-
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
 #else

src/util/src/ttokenizer.c

@@ -219,8 +219,13 @@ static SKeyword keywordTable[] = {
     {"PARTITIONS",   TK_PARTITIONS},
     {"TOPIC",        TK_TOPIC},
     {"TOPICS",       TK_TOPICS},
-    {"COMPACT",       TK_COMPACT},
-    {"MODIFY",       TK_MODIFY}
+    {"COMPACT",      TK_COMPACT},
+    {"MODIFY",       TK_MODIFY},
+    {"FUNCTION",     TK_FUNCTION},
+    {"FUNCTIONS",    TK_FUNCTIONS},
+    {"OUTPUTTYPE",   TK_OUTPUTTYPE},
+    {"AGGREGATE",    TK_AGGREGATE},
+    {"BUFSIZE",      TK_BUFSIZE},
 };
 
 static const char isIdChar[] = {
@@ -269,7 +274,7 @@ static int32_t tKeywordCode(const char* z, int n) {
   if (keywordHashTable == NULL) {
     return TK_ILLEGAL;
   }
-  
+
   SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
   return (pKey != NULL)? (*pKey)->type:TK_ID;
 }

src/vnode/src/vnodeRead.c

@@ -231,7 +231,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
     uint64_t qId = genQueryId();
-    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, &qId);
+    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId);
 
     SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code = code;

tests/examples/c/CMakeLists.txt

@@ -8,13 +8,13 @@ IF (TD_LINUX)
   ADD_EXECUTABLE(subscribe subscribe.c)
   TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread )
   ADD_EXECUTABLE(epoll epoll.c)
-  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread )
+  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()
 IF (TD_DARWIN)
   INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo demo.c)
-  TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
+  TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)
   ADD_EXECUTABLE(epoll epoll.c)
-  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread )
+  TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()

tests/script/general/parser/testSuite.sim

@@ -12,7 +12,7 @@ run general/parser/create_tb.sim
 run general/parser/dbtbnameValidate.sim
 run general/parser/fill.sim
 run general/parser/fill_stb.sim
-run general/parser/fill_us.sim
+#run general/parser/fill_us.sim               #
 run general/parser/first_last.sim
 run general/parser/import_commit1.sim
 run general/parser/import_commit2.sim
@@ -39,7 +39,6 @@ run general/parser/slimit1.sim
 run general/parser/slimit_alter_tags.sim
 run general/parser/tbnameIn.sim
 run general/parser/join.sim
-#run general/parser/join_multitables.sim
 run general/parser/join_multivnode.sim
 run general/parser/join_manyblocks.sim
 run general/parser/projection_limit_offset.sim
@@ -61,5 +60,9 @@ run general/parser/slimit_alter_tags.sim
 run general/parser/binary_escapeCharacter.sim
 run general/parser/between_and.sim
 run general/parser/last_cache.sim
+run general/parser/slimit_alter_tags.sim
+run general/parser/udf.sim
+run general/parser/udf_dll.sim
+run general/parser/udf_dll_stable.sim
 run general/parser/nestquery.sim
 run general/parser/precision_ns.sim

tests/script/sh/add_one.c

+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct SUdfInit{
+ int maybe_null;       /* 1 if function can return NULL */
+ int decimals;     /* for real functions */
+ long long length;       /* For string functions */
+ char  *ptr;            /* free pointer for function data */
+ int const_item;       /* 0 if result is independent of arguments */
+} SUdfInit;
+
+void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput,
+                        int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
+   int i;
+   int r = 0;
+   printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
+   if (itype == 4) {
+     for(i=0;i<numOfRows;++i) {
+       printf("input %d - %d", i, *((int *)data + i));
+       *((int *)dataOutput+i)=*((int *)data + i) + 1;
+       printf(", output %d\n", *((int *)dataOutput+i));
+       if (tsOutput) {
+         *(long long*)tsOutput=1000000;
+       }
+     }
+     *numOfOutput=numOfRows;
+
+     printf("add_one out, numOfOutput:%d\n", *numOfOutput);
+   }
+}
+
+