diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 55ca1174c91dd2a27049cc82733836344e73b90a..07f1d61fad81cd9653de257f47a7de309e664bf1 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -19,6 +19,7 @@ else fi # Dynamic directory + data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then @@ -29,25 +30,32 @@ fi data_link_dir="/usr/local/taos/data" log_link_dir="/usr/local/taos/log" - -cfg_install_dir="/etc/taos" +if [ "$osType" != "Darwin" ]; then + cfg_install_dir="/etc/taos" +else + cfg_install_dir="/usr/local/Cellar/tdengine/${verNumber}/taos" +fi if [ "$osType" != "Darwin" ]; then bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" -else - bin_link_dir="/usr/local/bin" - lib_link_dir="/usr/local/lib" - inc_link_dir="/usr/local/include" fi #install main path -install_main_dir="/usr/local/taos" +if [ "$osType" != "Darwin" ]; then + install_main_dir="/usr/local/taos" +else + install_main_dir="/usr/local/Cellar/tdengine/${verNumber}" +fi # old bin dir -bin_dir="/usr/local/taos/bin" +if [ "$osType" != "Darwin" ]; then + bin_dir="/usr/local/taos/bin" +else + bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin" +fi service_config_dir="/etc/systemd/system" @@ -59,12 +67,11 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi if [ "$osType" != "Darwin" ]; then - + if command -v sudo > /dev/null; then + csudo="sudo" + fi initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then @@ -137,18 +144,16 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo} rm -f ${bin_link_dir}/taos || : - if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/perfMonitor || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/set_core || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : fi - - ${csudo} rm -f ${bin_link_dir}/rmtaos || : - + ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin @@ -162,20 +167,17 @@ function install_bin() { ${csudo} chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : fi - + if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : - else - [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : fi } @@ -222,7 +224,7 @@ function install_jemalloc() { fi if [ -d /etc/ld.so.conf.d ]; then - ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf ${csudo} ldconfig else echo "/etc/ld.so.conf.d not found!" @@ -248,10 +250,8 @@ function install_lib() { fi else ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib - ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + install_jemalloc if [ "$osType" != "Darwin" ]; then @@ -261,10 +261,14 @@ function install_lib() { function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + fi ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* - ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + fi } function install_config() { @@ -272,29 +276,30 @@ function install_config() { if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/../cfg/taos.cfg ] && + ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + + if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + fi } function install_log() { - ${csudo} rm -rf ${log_dir} || : - if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - else - mkdir -p ${log_dir} && chmod 777 ${log_dir} + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + ${csudo} ln -s ${log_dir} ${install_main_dir}/log fi - - ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { - ${csudo} mkdir -p ${data_dir} - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${data_dir} + ${csudo} ln -s ${data_dir} ${install_main_dir}/data + fi } function install_connector() { @@ -309,7 +314,6 @@ function install_connector() { echo "WARNING: go connector not found, please check if want to use it!" fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } @@ -489,24 +493,24 @@ function install_TDengine() { else echo -e "${GREEN}Start to install TDEngine Client ...${NC}" fi - + install_main_path if [ "$osType" != "Darwin" ]; then install_data fi + install_log install_header install_lib install_connector install_examples - install_bin - + if [ "$osType" != "Darwin" ]; then install_service fi - + install_config if [ "$osType" != "Darwin" ]; then diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ac5b4f2ec2073c4d903cbd72aa1c149b81543588..4249155eab6284c54f206dfa4d2a8e82fb1be895 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -307,6 +307,7 @@ typedef struct { char * data; TAOS_ROW tsrow; TAOS_ROW urow; + bool dataConverted; int32_t* length; // length for each field for current row char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex* pColumnIndex; @@ -439,7 +440,7 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo); void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); +void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo, bool converted); void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock, bool convertNchar); void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index d86a1927466b263967abca1c20225ea26d1b53fb..e8cd9a257a7a946bb26da45376ff8b7d3f092d0b 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1522,6 +1522,7 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { pSql->isBind = true; pStmt->pSql = pSql; pStmt->last = STMT_INIT; + registerSqlObj(pSql); return pStmt; } @@ -1783,7 +1784,9 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { int taos_stmt_close(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_CHECK + if (pStmt == NULL || pStmt->taos == NULL) { + STMT_RET(TSDB_CODE_TSC_DISCONNECTED); + } if (!pStmt->isInsert) { SNormalStmt* normal = &pStmt->normal; if (normal->params != NULL) { @@ -1805,8 +1808,9 @@ int taos_stmt_close(TAOS_STMT* stmt) { pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); if (pStmt->pSql){ taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); + pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; } - pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; + taosArrayDestroy(pStmt->mtb.tags); tfree(pStmt->mtb.sqlstr); } @@ -1817,6 +1821,7 @@ int taos_stmt_close(TAOS_STMT* stmt) { } else { tscFreeSqlObj(pStmt->pSql); } + tfree(pStmt); STMT_RET(TSDB_CODE_SUCCESS); } @@ -1994,6 +1999,7 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) { return NULL; } TAOS_RES* result = pStmt->pSql; + pStmt->pSql = NULL; return result; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 31448881d31a9cafa57590bb492e5f11a4ec0c1c..da5bd7d3cc5bc752105e288bf5976fd5ed42a7f3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3486,6 +3486,27 @@ static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) { return true; } +bool groupbyTbname(SQueryInfo* pQueryInfo) { + if (pQueryInfo->groupbyExpr.columnInfo == NULL || + taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo) == 0) { + return false; + } + + size_t s = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + for (int32_t i = 0; i < s; i++) { + SColIndex* colIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); + if (colIndex->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + return true; + } + } + + return false; +} + + + + + static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) { int32_t startIdx = 0; int32_t aggUdf = 0; @@ -7114,6 +7135,35 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* } } + +int32_t validateFunctionFromUpstream(SQueryInfo* pQueryInfo, char* msg) { + const char* msg1 = "TWA/Diff/Derivative/Irate are not allowed to apply to super table without group by tbname"; + + int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo); + size_t upNum = taosArrayGetSize(pQueryInfo->pUpstream); + + for (int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + + int32_t f = pExpr->base.functionId; + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE || f == TSDB_FUNC_DIFF) { + for (int32_t j = 0; j < upNum; ++j) { + SQueryInfo* pUp = taosArrayGetP(pQueryInfo->pUpstream, j); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pUp, 0); + bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + if ((!isSTable) || groupbyTbname(pUp)) { + return TSDB_CODE_SUCCESS; + } + } + + return invalidOperationMsg(msg, msg1); + } + } + + return TSDB_CODE_SUCCESS; +} + + int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { const char* msg1 = "only one expression allowed"; const char* msg2 = "invalid expression in select clause"; @@ -8088,6 +8138,10 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); for (int32_t i = 0; i < num; ++i) { SSqlNode* p = taosArrayGetP(sub->pSubquery, i); + if (p->from == NULL) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if (p->from->type == SQL_NODE_FROM_TABLELIST) { int32_t code = getTableNameFromSqlNode(p, tableNameList, msgBuf, pSql); if (code != TSDB_CODE_SUCCESS) { @@ -8663,6 +8717,10 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return code; } + if ((code = validateFunctionFromUpstream(pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { + return code; + } + // updateFunctionInterBuf(pQueryInfo, false); updateLastScanOrderIfNeeded(pQueryInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4dd252b692abcf62ad210a88c86f94a689caa84b..bdebd14f571d1fdca0192c6636e8ade4073eb9bc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1892,7 +1892,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { return pRes->code; } - tscSetResRawPtr(pRes, pQueryInfo); + tscSetResRawPtr(pRes, pQueryInfo, pRes->dataConverted); } else { tscResetForNextRetrieve(pRes); } @@ -2807,7 +2807,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { - tscSetResRawPtr(pRes, pQueryInfo); + tscSetResRawPtr(pRes, pQueryInfo, pRes->dataConverted); } if (pSql->pSubscription != NULL) { @@ -3011,6 +3011,8 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } } + + tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index a8cd56204ef6af812ac4e27c610a3565a79827d1..eeb523b60d8ec70b0fe7260b3acbe9ccdedf626b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2435,7 +2435,11 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SSqlObj* pSql = pSchedMsg->ahandle; SPair* p = pSchedMsg->msg; - for(int32_t i = p->first; i < p->second; ++i) { + for (int32_t i = p->first; i < p->second; ++i) { + if (i >= pSql->subState.numOfSub) { + tfree(p); + return; + } SSqlObj* pSub = pSql->pSubs[i]; SRetrieveSupport* pSupport = pSub->param; @@ -2575,7 +2579,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; assert(numOfTasks >= 1); - int32_t num = (pState->numOfSub/numOfTasks) + 1; + int32_t num; + if (pState->numOfSub / numOfTasks == MAX_REQUEST_PER_TASK) { + num = MAX_REQUEST_PER_TASK; + } else { + num = pState->numOfSub / numOfTasks + 1; + } tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); for(int32_t j = 0; j < numOfTasks; ++j) { @@ -3402,6 +3411,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { } if (numOfRes == 0) { // no result any more, free all subquery objects + pSql->res.completed = true; freeJoinSubqueryObj(pSql); return; } @@ -3448,6 +3458,8 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes); memcpy(data, pData, bytes * numOfRes); + pRes->dataConverted = pRes1->dataConverted; + data += bytes * numOfRes; } @@ -3473,7 +3485,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { doArithmeticCalculate(pQueryInfo, pFilePage, rowSize, finalRowSize); pRes->data = pFilePage->data; - tscSetResRawPtr(pRes, pQueryInfo); + tscSetResRawPtr(pRes, pQueryInfo, pRes->dataConverted); } void tscBuildResFromSubqueries(SSqlObj *pSql) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 21e67e56e87da03afd400bea6321358541425667..1b4dff4b1b5a9c0b8053ed3f6289676c87ab0289 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -690,9 +690,13 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); } + + if (convertNchar) { + pRes->dataConverted = true; + } } -void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { +void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo, bool converted) { assert(pRes->numOfCols > 0); if (pRes->numOfRows == 0) { return; @@ -705,7 +709,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { pRes->length[i] = pInfo->field.bytes; offset += pInfo->field.bytes; - setResRawPtrImpl(pRes, pInfo, i, true); + setResRawPtrImpl(pRes, pInfo, i, converted ? false : true); } } @@ -3424,6 +3428,7 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { pRes->row = 0; pRes->numOfRows = 0; + pRes->dataConverted = false; } void tscInitResForMerge(SSqlRes* pRes) { diff --git a/src/connector/python/README.md b/src/connector/python/README.md index a5dc2b72dafbeef0bf53fc1768f6afc66e714699..95ef26e1f0e73cee7d47ecb6ece1d6a95d2f89d3 100644 --- a/src/connector/python/README.md +++ b/src/connector/python/README.md @@ -1,6 +1,7 @@ # TDengine Connector for Python -[TDengine] connector for Python enables python programs to access TDengine, using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. ## Install @@ -11,8 +12,417 @@ pip install ./TDengine/src/connector/python ## Source Code -[TDengine] connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). -## License - AGPL +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"pass it again_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 Keep same with [TDengine](https://github.com/taosdata/TDengine). diff --git a/src/connector/python/examples/bind-multi.py b/src/connector/python/examples/bind-multi.py new file mode 100644 index 0000000000000000000000000000000000000000..8530253aef58079e01f5eb71d8e12ab1649b7731 --- /dev/null +++ b/src/connector/python/examples/bind-multi.py @@ -0,0 +1,50 @@ +# encoding:UTF-8 +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt_multi" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() \ No newline at end of file diff --git a/src/connector/python/examples/bind-row.py b/src/connector/python/examples/bind-row.py new file mode 100644 index 0000000000000000000000000000000000000000..4ab9a9167ad23a6167c6586aac30ae6941dcee6d --- /dev/null +++ b/src/connector/python/examples/bind-row.py @@ -0,0 +1,57 @@ +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +# No need to explicitly close, but ok for you +# result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) + +# No need to explicitly close, but ok for you +# result.close() +# stmt.close() +# conn.close() diff --git a/src/connector/python/examples/insert-lines.py b/src/connector/python/examples/insert-lines.py new file mode 100644 index 0000000000000000000000000000000000000000..0096b7e8cdf1328ee78805a1ee3134ad7cdfc447 --- /dev/null +++ b/src/connector/python/examples/insert-lines.py @@ -0,0 +1,22 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000ns', +] +conn.insert_lines(lines) +print("inserted") + +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) + + +conn.execute("drop database if exists %s" % dbname) diff --git a/src/connector/python/examples/pep-249.py b/src/connector/python/examples/pep-249.py new file mode 100644 index 0000000000000000000000000000000000000000..971a3c401f00b982096b8d429f65bce73cca4760 --- /dev/null +++ b/src/connector/python/examples/pep-249.py @@ -0,0 +1,9 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) diff --git a/src/connector/python/examples/query-async.py b/src/connector/python/examples/query-async.py new file mode 100644 index 0000000000000000000000000000000000000000..b600b796974e47d5e5fc7d88998e95ba46bb92cd --- /dev/null +++ b/src/connector/python/examples/query-async.py @@ -0,0 +1,62 @@ +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + # should explicitly close the result in fetch completed or cause error + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + # explicitly close result while query failed + result.close() + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + # conn.close() + + +if __name__ == "__main__": + test_query(connect()) \ No newline at end of file diff --git a/src/connector/python/examples/query-objectively.py b/src/connector/python/examples/query-objectively.py new file mode 100644 index 0000000000000000000000000000000000000000..104347cbf91e29e62fef26477b475053a8b8bc3e --- /dev/null +++ b/src/connector/python/examples/query-objectively.py @@ -0,0 +1,12 @@ +import taos + +conn = taos.connect() +conn.execute("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +conn.execute("drop database pytest") diff --git a/src/connector/python/examples/subscribe-async.py b/src/connector/python/examples/subscribe-async.py new file mode 100644 index 0000000000000000000000000000000000000000..3782ce5505152e78838406e313094eb911bea4a2 --- /dev/null +++ b/src/connector/python/examples/subscribe-async.py @@ -0,0 +1,43 @@ +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + # sub.close() + + conn.execute("drop database if exists %s" % dbname) + # conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + # conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) diff --git a/src/connector/python/examples/subscribe-sync.py b/src/connector/python/examples/subscribe-sync.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7f65f460280924ed3a577fe55b975fbf12c1a3 --- /dev/null +++ b/src/connector/python/examples/subscribe-sync.py @@ -0,0 +1,53 @@ +import taos +import random + +conn = taos.connect() +dbname = "pytest_taos_subscribe" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.execute("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + +sub = conn.subscribe(False, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +sub.close(True) +print("# keep progress consume") +sub = conn.subscribe(False, "test", "select * from log", 1000) +result = sub.consume() +rows = result.fetch_all() +# consume from latest subscription needs root privilege(for /var/lib/taos). +assert result.row_count == 0 +print("## consumed ", len(rows), "rows") + +print("# consume with a stop condition") +for i in range(10): + conn.execute("insert into log values(now, %d)" % random.randint(0, 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +# sub.close() + +conn.execute("drop database if exists %s" % dbname) +# conn.close() diff --git a/src/connector/python/pyproject.toml b/src/connector/python/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..a8099199563a0e5957a7d69e75bab65cca6d17db --- /dev/null +++ b/src/connector/python/pyproject.toml @@ -0,0 +1,27 @@ +[tool.poetry] +name = "taos" +version = "2.1.0" +description = "TDengine connector for python" +authors = ["Taosdata Inc. "] +license = "AGPL-3.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^2.7 || ^3.4" +typing = "*" + +[tool.poetry.dev-dependencies] +pytest = [ + { version = "^4.6", python = "^2.7" }, + { version = "^6.2", python = "^3.7" } +] +pdoc = { version = "^7.1.1", python = "^3.7" } +mypy = { version = "^0.910", python = "^3.6" } +black = { version = "^21.7b0", python = "^3.6" } + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 119 diff --git a/src/connector/python/setup.py b/src/connector/python/setup.py index 284861ca87dd77b1bc6799ad4cb32ff4c489e239..b7e10001737bc40c04173ea4a65e95248965ffda 100644 --- a/src/connector/python/setup.py +++ b/src/connector/python/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.11", + version="2.1.0", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/taos/__init__.py b/src/connector/python/taos/__init__.py index 52c6db311ecc4c2f944372ae3334fdc58cb6e779..75138eade3d60f7894d814babe58cec7aecc9a20 100644 --- a/src/connector/python/taos/__init__.py +++ b/src/connector/python/taos/__init__.py @@ -1,20 +1,478 @@ +# encoding:UTF-8 +""" +# TDengine Connector for Python -from .connection import TDengineConnection -from .cursor import TDengineCursor +[TDengine](https://github.com/taosdata/TDengine) connector for Python enables python programs to access TDengine, + using an API which is compliant with the Python DB API 2.0 (PEP-249). It uses TDengine C client library for client server communications. -# For some reason, the following is needed for VS Code (through PyLance) to +## Install + +```sh +git clone --depth 1 https://github.com/taosdata/TDengine.git +pip install ./TDengine/src/connector/python +``` + +## Source Code + +[TDengine](https://github.com/taosdata/TDengine) connector for Python source code is hosted on [GitHub](https://github.com/taosdata/TDengine/tree/develop/src/connector/python). + +## Examples + +### Query with PEP-249 API + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +cursor.close() +conn.close() +``` + +### Query with objective API + +```python +import taos + +conn = taos.connect() +conn.exec("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +result.close() +conn.exec("drop database pytest") +conn.close() +``` + +### Query with async API + +```python +from taos import * +from ctypes import * +import time + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) +``` + +### Statement API - Bind row after row + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_bind_params(16) +params[0].timestamp(1626861392589) +params[1].bool(True) +params[2].null() +params[3].tinyint(2) +params[4].smallint(3) +params[5].int(4) +params[6].bigint(5) +params[7].tinyint_unsigned(6) +params[8].smallint_unsigned(7) +params[9].int_unsigned(8) +params[10].bigint_unsigned(9) +params[11].float(10.1) +params[12].double(10.11) +params[13].binary("hello") +params[14].nchar("stmt") +params[15].timestamp(1626861392589) +stmt.bind_param(params) + +params[0].timestamp(1626861392590) +params[15].null() +stmt.bind_param(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 2 +result.close() + +result = conn.query("select * from log") + +for row in result: + print(row) +result.close() +stmt.close() +conn.close() + +``` + +### Statement API - Bind multi rows + +```python +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.exec( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \\ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \\ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \\ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() +``` + +### Statement API - Subscribe + +```python +import taos + +conn = taos.connect() +dbname = "pytest_taos_subscribe_callback" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s" % dbname) +conn.select_db(dbname) +conn.exec("create table if not exists log(ts timestamp, n int)") +for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + +sub = conn.subscribe(True, "test", "select * from log", 1000) +print("# consume from begin") +for ts, n in sub.consume(): + print(ts, n) + +print("# consume new data") +for i in range(5): + conn.exec("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + +print("# consume with a stop condition") +for i in range(10): + conn.exec("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + +sub.close() + +conn.exec("drop database if exists %s" % dbname) +conn.close() +``` + +### Statement API - Subscribe asynchronously with callback + +```python +from taos import * +from ctypes import * + +import time + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("# fetch in callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.exec("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe_callback(connect()) + +``` + +### Statement API - Stream + +```python +from taos import * +from ctypes import * + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.exec("drop database if exists %s" % dbname) + conn.exec("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.exec("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.exec("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.exec("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.exec("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) +``` + +### Insert with line protocol + +```python +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.exec("drop database if exists %s" % dbname) +conn.exec("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) +print("inserted") + +lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', +] +conn.insert_lines(lines) + +result = conn.query("show tables") +for row in result: + print(row) +result.close() + + +conn.exec("drop database if exists %s" % dbname) +conn.close() + +``` + +## License - AGPL-3.0 + +Keep same with [TDengine](https://github.com/taosdata/TDengine). +""" +from .connection import TaosConnection + +# For some reason, the following is needed for VS Code (through PyLance) to # recognize that "error" is a valid module of the "taos" package. -from .error import ProgrammingError +from .error import * +from .bind import * +from .field import * +from .cursor import * +from .result import * +from .statement import * +from .subscription import * + +try: + import importlib.metadata + + __version__ = importlib.metadata.version("taos") +except: + None # Globals threadsafety = 0 -paramstyle = 'pyformat' - -__all__ = ['connection', 'cursor'] +paramstyle = "pyformat" +__all__ = [ + # functions + "connect", + "new_bind_param", + "new_bind_params", + "new_multi_binds", + "new_multi_bind", + # objects + "TaosBind", + "TaosConnection", + "TaosCursor", + "TaosResult", + "TaosRows", + "TaosRow", + "TaosStmt", + "PrecisionEnum", +] def connect(*args, **kwargs): - """ Function to return a TDengine connector object + # type: (..., ...) -> TaosConnection + """Function to return a TDengine connector object Current supporting keyword parameters: @dsn: Data source name as string @@ -25,4 +483,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) + return TaosConnection(*args, **kwargs) diff --git a/src/connector/python/taos/bind.py b/src/connector/python/taos/bind.py new file mode 100644 index 0000000000000000000000000000000000000000..083ddc99aea8dc6f39b1f22ac5f77d2584a2fe69 --- /dev/null +++ b/src/connector/python/taos/bind.py @@ -0,0 +1,437 @@ +# encoding:UTF-8 +import ctypes +from .constants import FieldType +from .error import * +from .precision import * +from datetime import datetime +from ctypes import * +import sys + +_datetime_epoch = datetime.utcfromtimestamp(0) + +def _is_not_none(obj): + return obj != None + +class TaosBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_size_t)), + ("is_null", POINTER(c_int)), + ("is_unsigned", c_int), + ("error", POINTER(c_int)), + ("u", c_int64), + ("allocated", c_int), + ] + + def null(self): + self.buffer_type = FieldType.C_NULL + self.is_null = pointer(c_int(1)) + + def bool(self, value): + self.buffer_type = FieldType.C_BOOL + self.buffer = cast(pointer(c_bool(value)), c_void_p) + self.buffer_length = sizeof(c_bool) + + def tinyint(self, value): + self.buffer_type = FieldType.C_TINYINT + self.buffer = cast(pointer(c_int8(value)), c_void_p) + self.buffer_length = sizeof(c_int8) + + def smallint(self, value): + self.buffer_type = FieldType.C_SMALLINT + self.buffer = cast(pointer(c_int16(value)), c_void_p) + self.buffer_length = sizeof(c_int16) + + def int(self, value): + self.buffer_type = FieldType.C_INT + self.buffer = cast(pointer(c_int32(value)), c_void_p) + self.buffer_length = sizeof(c_int32) + + def bigint(self, value): + self.buffer_type = FieldType.C_BIGINT + self.buffer = cast(pointer(c_int64(value)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def float(self, value): + self.buffer_type = FieldType.C_FLOAT + self.buffer = cast(pointer(c_float(value)), c_void_p) + self.buffer_length = sizeof(c_float) + + def double(self, value): + self.buffer_type = FieldType.C_DOUBLE + self.buffer = cast(pointer(c_double(value)), c_void_p) + self.buffer_length = sizeof(c_double) + + def binary(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_BINARY + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def timestamp(self, value, precision=PrecisionEnum.Milliseconds): + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + ts = value + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + ts = int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + ts = int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(pointer(c_int64(ts)), c_void_p) + self.buffer_length = sizeof(c_int64) + + def nchar(self, value): + buffer = None + length = 0 + if isinstance(value, str): + bytes = value.encode("utf-8") + buffer = create_string_buffer(bytes) + length = len(bytes) + else: + buffer = value + length = len(value) + self.buffer_type = FieldType.C_NCHAR + self.buffer = cast(buffer, c_void_p) + self.buffer_length = length + self.length = pointer(c_size_t(self.buffer_length)) + + def tinyint_unsigned(self, value): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer = cast(pointer(c_uint8(value)), c_void_p) + self.buffer_length = sizeof(c_uint8) + + def smallint_unsigned(self, value): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer = cast(pointer(c_uint16(value)), c_void_p) + self.buffer_length = sizeof(c_uint16) + + def int_unsigned(self, value): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer = cast(pointer(c_uint32(value)), c_void_p) + self.buffer_length = sizeof(c_uint32) + + def bigint_unsigned(self, value): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer = cast(pointer(c_uint64(value)), c_void_p) + self.buffer_length = sizeof(c_uint64) + + +def _datetime_to_timestamp(value, precision): + # type: (datetime | float | int | str | c_int64, PrecisionEnum) -> c_int64 + if value is None: + return FieldType.C_BIGINT_NULL + if type(value) is datetime: + if precision == PrecisionEnum.Milliseconds: + return int(round((value - _datetime_epoch).total_seconds() * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round((value - _datetime_epoch).total_seconds() * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif type(value) is float: + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("time float do not support nanosecond precision") + elif isinstance(value, int) and not isinstance(value, bool): + return c_int64(value) + elif isinstance(value, str): + value = datetime.fromisoformat(value) + if precision == PrecisionEnum.Milliseconds: + return int(round(value * 1000)) + elif precision == PrecisionEnum.Microseconds: + return int(round(value * 10000000)) + else: + raise PrecisionError("datetime do not support nanosecond precision") + elif isinstance(value, c_int64): + return value + return FieldType.C_BIGINT_NULL + + +class TaosMultiBind(ctypes.Structure): + _fields_ = [ + ("buffer_type", c_int), + ("buffer", c_void_p), + ("buffer_length", c_size_t), + ("length", POINTER(c_int32)), + ("is_null", c_char_p), + ("num", c_int), + ] + + def null(self, num): + self.buffer_type = FieldType.C_NULL + self.is_null = cast((c_char * num)(*[1 for _ in range(num)]), c_char_p) + self.buffer = c_void_p(None) + self.num = num + + def bool(self, values): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BOOL_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + self.buffer_type = FieldType.C_BOOL + self.buffer_length = sizeof(c_bool) + + def tinyint(self, values): + self.buffer_type = FieldType.C_TINYINT + self.buffer_length = sizeof(c_int8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_NULL for v in values]) + + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint(self, values): + self.buffer_type = FieldType.C_SMALLINT + self.buffer_length = sizeof(c_int16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int(self, values): + self.buffer_type = FieldType.C_INT + self.buffer_length = sizeof(c_int32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint(self, values): + self.buffer_type = FieldType.C_BIGINT + self.buffer_length = sizeof(c_int64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def float(self, values): + self.buffer_type = FieldType.C_FLOAT + self.buffer_length = sizeof(c_float) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_float * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_FLOAT_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def double(self, values): + self.buffer_type = FieldType.C_DOUBLE + self.buffer_length = sizeof(c_double) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_double * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_DOUBLE_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def _str_to_buffer(self, values): + self.num = len(values) + is_null = [1 if v == None else 0 for v in values] + self.is_null = cast((c_byte * self.num)(*is_null), c_char_p) + + if sum(is_null) == self.num: + self.length = (c_int32 * len(values))(0 * self.num) + return + if sys.version_info < (3, 0): + _bytes = [bytes(value) if value is not None else None for value in values] + buffer_length = max(len(b) + 1 for b in _bytes if b is not None) + buffers = [ + create_string_buffer(b, buffer_length) if b is not None else create_string_buffer(buffer_length) + for b in _bytes + ] + buffer_all = b''.join(v[:] for v in buffers) + self.buffer = cast(c_char_p(buffer_all), c_void_p) + else: + _bytes = [value.encode("utf-8") if value is not None else None for value in values] + buffer_length = max(len(b) for b in _bytes if b is not None) + self.buffer = cast( + c_char_p( + b"".join( + [ + create_string_buffer(b, buffer_length) + if b is not None + else create_string_buffer(buffer_length) + for b in _bytes + ] + ) + ), + c_void_p, + ) + self.length = (c_int32 * len(values))(*[len(b) if b is not None else 0 for b in _bytes]) + self.buffer_length = buffer_length + def binary(self, values): + self.buffer_type = FieldType.C_BINARY + self._str_to_buffer(values) + + def timestamp(self, values, precision=PrecisionEnum.Milliseconds): + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_int64 * len(values) + buffer = buffer_type(*[_datetime_to_timestamp(value, precision) for value in values]) + + self.buffer_type = FieldType.C_TIMESTAMP + self.buffer = cast(buffer, c_void_p) + self.buffer_length = sizeof(c_int64) + self.num = len(values) + + def nchar(self, values): + # type: (list[str]) -> None + self.buffer_type = FieldType.C_NCHAR + self._str_to_buffer(values) + + def tinyint_unsigned(self, values): + self.buffer_type = FieldType.C_TINYINT_UNSIGNED + self.buffer_length = sizeof(c_uint8) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint8 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_TINYINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def smallint_unsigned(self, values): + self.buffer_type = FieldType.C_SMALLINT_UNSIGNED + self.buffer_length = sizeof(c_uint16) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint16 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_SMALLINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def int_unsigned(self, values): + self.buffer_type = FieldType.C_INT_UNSIGNED + self.buffer_length = sizeof(c_uint32) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint32 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_INT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + def bigint_unsigned(self, values): + self.buffer_type = FieldType.C_BIGINT_UNSIGNED + self.buffer_length = sizeof(c_uint64) + + try: + buffer = cast(values, c_void_p) + except: + buffer_type = c_uint64 * len(values) + try: + buffer = buffer_type(*values) + except: + buffer = buffer_type(*[v if v is not None else FieldType.C_BIGINT_UNSIGNED_NULL for v in values]) + self.buffer = cast(buffer, c_void_p) + self.num = len(values) + + +def new_bind_param(): + # type: () -> TaosBind + return TaosBind() + + +def new_bind_params(size): + # type: (int) -> Array[TaosBind] + return (TaosBind * size)() + + +def new_multi_bind(): + # type: () -> TaosMultiBind + return TaosMultiBind() + + +def new_multi_binds(size): + # type: (int) -> Array[TaosMultiBind] + return (TaosMultiBind * size)() diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index d9db1272ef2802463a891d3c626b66111f805975..42dac3c2e8f2647193d88b250db083b64cac3b6b 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -1,295 +1,839 @@ +# encoding:UTF-8 + import ctypes -from .constants import FieldType -from .error import * -import math -import datetime import platform +import sys +from ctypes import * +try: + from typing import Any +except: + pass + +from .error import * +from .bind import * +from .field import * + + +# stream callback +stream_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p) +stream_callback2_type = CFUNCTYPE(None, c_void_p) + +# C interface class +class TaosOption: + Locale = (0,) + Charset = (1,) + Timezone = (2,) + ConfigDir = (3,) + ShellActivityTimer = (4,) + MaxOptions = (5,) + + +def _load_taos_linux(): + return ctypes.CDLL("libtaos.so") + + +def _load_taos_darwin(): + return ctypes.CDLL("libtaos.dylib") + +def _load_taos_windows(): + return ctypes.windll.LoadLibrary("taos") -def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=milli/1000.0) +def _load_taos(): + load_func = { + "Linux": _load_taos_linux, + "Darwin": _load_taos_darwin, + "Windows": _load_taos_windows, + } + try: + return load_func[platform.system()]() + except: + raise InterfaceError('unsupported platform or failed to load taos client library') -def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=micro / 1000000.0) +_libtaos = _load_taos() -def _convert_nanosecond_to_datetime(nanosec): - return nanosec +_libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) +_libtaos.taos_init.restype = None +_libtaos.taos_connect.restype = ctypes.c_void_p +_libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) +_libtaos.taos_errstr.restype = ctypes.c_char_p +_libtaos.taos_subscribe.restype = ctypes.c_void_p +_libtaos.taos_consume.restype = ctypes.c_void_p +_libtaos.taos_fetch_lengths.restype = ctypes.POINTER(ctypes.c_int) +_libtaos.taos_free_result.restype = None +_libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +try: + _libtaos.taos_stmt_errstr.restype = c_char_p +except AttributeError: + None +finally: + None -def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row +_libtaos.taos_options.restype = None + + +def taos_options(option, *args): + # type: (TaosOption, Any) -> None + _libtaos.taos_options(option, *args) + + +def taos_init(): + # type: () -> None """ - _timestamp_converter = _convert_millisecond_to_datetime - if precision == FieldType.C_TIMESTAMP_MILLI: - _timestamp_converter = _convert_millisecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_MICRO: - _timestamp_converter = _convert_microsecond_to_datetime - elif precision == FieldType.C_TIMESTAMP_NANO: - _timestamp_converter = _convert_nanosecond_to_datetime - else: - raise DatabaseError("Unknown precision returned from database") + C: taos_init + """ + _libtaos.taos_init() + - return [ - None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_int64))[ - :abs(num_of_rows)]] +_libtaos.taos_cleanup.restype = None -def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bool row to python row +def taos_cleanup(): + # type: () -> None + """Cleanup workspace.""" + _libtaos.taos_cleanup() + + +_libtaos.taos_get_client_info.restype = c_char_p + + +def taos_get_client_info(): + # type: () -> str + """Get client version info. + 获取客户端版本信息。 """ - return [ - None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_byte))[ - :abs(num_of_rows)]] + return _libtaos.taos_get_client_info().decode() + + +_libtaos.taos_get_server_info.restype = c_char_p +_libtaos.taos_get_server_info.argtypes = (c_void_p,) + + +def taos_get_server_info(connection): + # type: (c_void_p) -> str + return _libtaos.taos_get_server_info(connection).decode() + + +_libtaos.taos_close.restype = None +_libtaos.taos_close.argtypes = (c_void_p,) + + +def taos_close(connection): + # type: (c_void_p) -> None + """Close the TAOS* connection""" + _libtaos.taos_close(connection) + + +_libtaos.taos_connect.restype = c_void_p +_libtaos.taos_connect.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + +def taos_connect(host=None, user="root", password="taosdata", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p + """Create TDengine database connection. + 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： -def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - password: user password / 用户密码 + - db: database name (optional) + - port: server port + + @rtype: c_void_p, TDengine handle """ - return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + # password + try: + _password = c_char_p(password.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") -def _crow_tinyint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C tinyint row to python row + # port + try: + _port = c_uint16(port) + except TypeError: + raise TypeError("port is expected as an uint16") + + connection = cast(_libtaos.taos_connect(_host, _user, _password, _db, _port), c_void_p) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_connect_auth.restype = c_void_p +_libtaos.taos_connect_auth.argtypes = c_char_p, c_char_p, c_char_p, c_char_p, c_uint16 + + +def taos_connect_auth(host=None, user="root", auth="", db=None, port=0): + # type: (None|str, str, str, None|str, int) -> c_void_p """ - return [ - None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ubyte))[ - :abs(num_of_rows)]] + 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： + - host: server hostname/FQDN, TDengine管理主节点的FQDN + - user: user name/用户名 + - auth: base64 encoded auth token + - db: database name (optional) + - port: server port -def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row + @rtype: c_void_p, TDengine handle """ - return [ - None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_short))[ - :abs(num_of_rows)]] + # host + try: + _host = c_char_p(host.encode("utf-8")) if host is not None else None + except AttributeError: + raise AttributeError("host is expected as a str") + # user + try: + _user = c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # auth + try: + _auth = c_char_p(auth.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = c_char_p(db.encode("utf-8")) if db is not None else None + except AttributeError: + raise AttributeError("db is expected as a str") + + # port + try: + _port = c_int(port) + except TypeError: + raise TypeError("port is expected as an int") + + connection = c_void_p(_libtaos.taos_connect_auth(_host, _user, _auth, _db, _port)) + + if connection.value is None: + raise ConnectionError("connect to TDengine failed") + return connection + + +_libtaos.taos_query.restype = c_void_p +_libtaos.taos_query.argtypes = c_void_p, c_char_p + + +def taos_query(connection, sql): + # type: (c_void_p, str) -> c_void_p + """Run SQL + + - sql: str, sql string to run + + @return: TAOS_RES*, result pointer -def _crow_smallint_unsigned_to_python( - data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C smallint row to python row """ - return [ - None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_ushort))[ - :abs(num_of_rows)]] + try: + ptr = c_char_p(sql.encode("utf-8")) + res = c_void_p(_libtaos.taos_query(connection, ptr)) + errno = taos_errno(res) + if errno != 0: + errstr = taos_errstr(res) + taos_free_result(res) + raise ProgrammingError(errstr, errno) + return res + except AttributeError: + raise AttributeError("sql is expected as a string") + + +async_query_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_query_a.restype = None +_libtaos.taos_query_a.argtypes = c_void_p, c_char_p, async_query_callback_type, c_void_p + + +def taos_query_a(connection, sql, callback, param): + # type: (c_void_p, str, async_query_callback_type, c_void_p) -> c_void_p + _libtaos.taos_query_a(connection, c_char_p(sql.encode("utf-8")), async_query_callback_type(callback), param) + + +async_fetch_rows_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_int) +_libtaos.taos_fetch_rows_a.restype = None +_libtaos.taos_fetch_rows_a.argtypes = c_void_p, async_fetch_rows_callback_type, c_void_p + +def taos_fetch_rows_a(result, callback, param): + # type: (c_void_p, async_fetch_rows_callback_type, c_void_p) -> c_void_p + _libtaos.taos_fetch_rows_a(result, async_fetch_rows_callback_type(callback), param) -def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row + +def taos_affected_rows(result): + # type: (c_void_p) -> c_int + """The affected rows after runing query""" + return _libtaos.taos_affected_rows(result) + + +subscribe_callback_type = CFUNCTYPE(None, c_void_p, c_void_p, c_void_p, c_int) +_libtaos.taos_subscribe.restype = c_void_p +# _libtaos.taos_subscribe.argtypes = c_void_p, c_int, c_char_p, c_char_p, subscribe_callback_type, c_void_p, c_int + + +def taos_subscribe(connection, restart, topic, sql, interval, callback=None, param=None): + # type: (c_void_p, bool, str, str, c_int, subscribe_callback_type, c_void_p | None) -> c_void_p + """Create a subscription + @restart boolean, + @sql string, sql statement for data query, must be a 'select' statement. + @topic string, name of this subscription """ - return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + if callback != None: + callback = subscribe_callback_type(callback) + if param != None: + param = c_void_p(param) + return c_void_p( + _libtaos.taos_subscribe( + connection, + 1 if restart else 0, + c_char_p(topic.encode("utf-8")), + c_char_p(sql.encode("utf-8")), + callback or None, + param, + interval, + ) + ) + + +_libtaos.taos_consume.restype = c_void_p +_libtaos.taos_consume.argstype = c_void_p, + + +def taos_consume(sub): + """Consume data of a subscription""" + return c_void_p(_libtaos.taos_consume(sub)) + + +_libtaos.taos_unsubscribe.restype = None +_libtaos.taos_unsubscribe.argstype = c_void_p, c_int + + +def taos_unsubscribe(sub, keep_progress): + """Cancel a subscription""" + _libtaos.taos_unsubscribe(sub, 1 if keep_progress else 0) + + +def taos_use_result(result): + """Use result after calling self.query, it's just for 1.6.""" + fields = [] + pfields = taos_fetch_fields_raw(result) + for i in range(taos_field_count(result)): + fields.append( + { + "name": pfields[i].name, + "bytes": pfields[i].bytes, + "type": pfields[i].type, + } + ) + + return fields + + +_libtaos.taos_fetch_block.restype = c_int +_libtaos.taos_fetch_block.argtypes = c_void_p, c_void_p + + +def taos_fetch_block_raw(result): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + return pblock, abs(num_of_rows) + + +def taos_fetch_block(result, fields=None, field_count=None): + pblock = ctypes.c_void_p(0) + num_of_rows = _libtaos.taos_fetch_block(result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + precision = taos_result_precision(result) + if fields == None: + fields = taos_fetch_fields(result) + if field_count == None: + field_count = taos_field_count(result) + blocks = [None] * field_count + fieldLen = taos_fetch_lengths(result, field_count) + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]["type"] not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i]["type"]](data, num_of_rows, fieldLen[i], precision) + + return blocks, abs(num_of_rows) + + +_libtaos.taos_fetch_row.restype = c_void_p +_libtaos.taos_fetch_row.argtypes = (c_void_p,) + + +def taos_fetch_row_raw(result): + # type: (c_void_p) -> c_void_p + row = c_void_p(_libtaos.taos_fetch_row(result)) + if row: + return row + return None + + +def taos_fetch_row(result, fields): + # type: (c_void_p, Array[TaosField]) -> tuple(c_void_p, int) + pblock = ctypes.c_void_p(0) + pblock = taos_fetch_row_raw(result) + if pblock: + num_of_rows = 1 + precision = taos_result_precision(result) + field_count = taos_field_count(result) + blocks = [None] * field_count + field_lens = taos_fetch_lengths(result, field_count) + for i in range(field_count): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = [None] + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, num_of_rows, field_lens[i], precision) + else: + return None, 0 + return blocks, abs(num_of_rows) + + +_libtaos.taos_free_result.argtypes = (c_void_p,) -def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C int row to python row +def taos_free_result(result): + # type: (c_void_p) -> None + if result != None: + _libtaos.taos_free_result(result) + + +_libtaos.taos_field_count.restype = c_int +_libtaos.taos_field_count.argstype = (c_void_p,) + + +def taos_field_count(result): + # type: (c_void_p) -> int + return _libtaos.taos_field_count(result) + + +def taos_num_fields(result): + # type: (c_void_p) -> int + return _libtaos.taos_num_fields(result) + + +_libtaos.taos_fetch_fields.restype = c_void_p +_libtaos.taos_fetch_fields.argstype = (c_void_p,) + + +def taos_fetch_fields_raw(result): + # type: (c_void_p) -> c_void_p + return c_void_p(_libtaos.taos_fetch_fields(result)) + + +def taos_fetch_fields(result): + # type: (c_void_p) -> TaosFields + fields = taos_fetch_fields_raw(result) + count = taos_field_count(result) + return TaosFields(fields, count) + + +def taos_fetch_lengths(result, field_count=None): + # type: (c_void_p, int) -> Array[int] + """Make sure to call taos_fetch_row or taos_fetch_block before fetch_lengths""" + lens = _libtaos.taos_fetch_lengths(result) + if field_count == None: + field_count = taos_field_count(result) + if not lens: + raise OperationalError("field length empty, use taos_fetch_row/block before it") + return lens[:field_count] + + +def taos_result_precision(result): + # type: (c_void_p) -> c_int + return _libtaos.taos_result_precision(result) + + +_libtaos.taos_errno.restype = c_int +_libtaos.taos_errno.argstype = (c_void_p,) + + +def taos_errno(result): + # type: (ctypes.c_void_p) -> c_int + """Return the error number.""" + return _libtaos.taos_errno(result) + + +_libtaos.taos_errstr.restype = c_char_p +_libtaos.taos_errstr.argstype = (c_void_p,) + + +def taos_errstr(result=c_void_p(None)): + # type: (ctypes.c_void_p) -> str + """Return the error styring""" + return _libtaos.taos_errstr(result).decode("utf-8") + + +_libtaos.taos_stop_query.restype = None +_libtaos.taos_stop_query.argstype = (c_void_p,) + + +def taos_stop_query(result): + # type: (ctypes.c_void_p) -> None + """Stop current query""" + return _libtaos.taos_stop_query(result) + + +_libtaos.taos_load_table_info.restype = c_int +_libtaos.taos_load_table_info.argstype = (c_void_p, c_char_p) + + +def taos_load_table_info(connection, tables): + # type: (ctypes.c_void_p, str) -> None + """Stop current query""" + errno = _libtaos.taos_load_table_info(connection, c_char_p(tables.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + raise OperationalError(msg, errno) + + +_libtaos.taos_validate_sql.restype = c_int +_libtaos.taos_validate_sql.argstype = (c_void_p, c_char_p) + + +def taos_validate_sql(connection, sql): + # type: (ctypes.c_void_p, str) -> None | str + """Get taosd server info""" + errno = _libtaos.taos_validate_sql(connection, ctypes.c_char_p(sql.encode("utf-8"))) + if errno != 0: + msg = taos_errstr() + return msg + return None + + +_libtaos.taos_print_row.restype = c_int +_libtaos.taos_print_row.argstype = (c_char_p, c_void_p, c_void_p, c_int) + + +def taos_print_row(row, fields, num_fields, buffer_size=4096): + # type: (ctypes.c_void_p, ctypes.c_void_p | TaosFields, int, int) -> str + """Print an row to string""" + p = ctypes.create_string_buffer(buffer_size) + if isinstance(fields, TaosFields): + _libtaos.taos_print_row(p, row, fields.as_ptr(), num_fields) + else: + _libtaos.taos_print_row(p, row, fields, num_fields) + if p: + return p.value.decode("utf-8") + raise OperationalError("taos_print_row failed") + + +_libtaos.taos_select_db.restype = c_int +_libtaos.taos_select_db.argstype = (c_void_p, c_char_p) + + +def taos_select_db(connection, db): + # type: (ctypes.c_void_p, str) -> None + """Select database, eq to sql: use """ + res = _libtaos.taos_select_db(connection, ctypes.c_char_p(db.encode("utf-8"))) + if res != 0: + raise DatabaseError("select database error", res) + + +try: + _libtaos.taos_open_stream.restype = c_void_p + _libtaos.taos_open_stream.argstype = c_void_p, c_char_p, stream_callback_type, c_int64, c_void_p, Any +except: + pass + + +def taos_open_stream(connection, sql, callback, stime=0, param=None, callback2=None): + # type: (ctypes.c_void_p, str, stream_callback_type, c_int64, c_void_p, c_void_p) -> ctypes.pointer + if callback2 != None: + callback2 = stream_callback2_type(callback2) + """Open an stream""" + return c_void_p( + _libtaos.taos_open_stream( + connection, ctypes.c_char_p(sql.encode("utf-8")), stream_callback_type(callback), stime, param, callback2 + ) + ) + + +_libtaos.taos_close_stream.restype = None +_libtaos.taos_close_stream.argstype = (c_void_p,) + + +def taos_close_stream(stream): + # type: (c_void_p) -> None + """Open an stream""" + return _libtaos.taos_close_stream(stream) + + +_libtaos.taos_stmt_init.restype = c_void_p +_libtaos.taos_stmt_init.argstype = (c_void_p,) + + +def taos_stmt_init(connection): + # type: (c_void_p) -> (c_void_p) + """Create a statement query + @param(connection): c_void_p TAOS* + @rtype: c_void_p, *TAOS_STMT """ - return [ - None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint))[ - :abs(num_of_rows)]] + return c_void_p(_libtaos.taos_stmt_init(connection)) -def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row +_libtaos.taos_stmt_prepare.restype = c_int +_libtaos.taos_stmt_prepare.argstype = (c_void_p, c_char_p, c_int) + + +def taos_stmt_prepare(stmt, sql): + # type: (ctypes.c_void_p, str) -> None + """Prepare a statement query + @stmt: c_void_p TAOS_STMT* """ - return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + buffer = sql.encode("utf-8") + res = _libtaos.taos_stmt_prepare(stmt, ctypes.c_char_p(buffer), len(buffer)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +_libtaos.taos_stmt_close.restype = c_int +_libtaos.taos_stmt_close.argstype = (c_void_p,) -def _crow_bigint_unsigned_to_python( - data, - num_of_rows, - nbytes=None, - precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C bigint row to python row + +def taos_stmt_close(stmt): + # type: (ctypes.c_void_p) -> None + """Close a statement query + @stmt: c_void_p TAOS_STMT* """ - return [ - None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER( - ctypes.c_uint64))[ - :abs(num_of_rows)]] + res = _libtaos.taos_stmt_close(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_errstr.restype = c_char_p + _libtaos.taos_stmt_errstr.argstype = (c_void_p,) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_errstr" % taos_get_client_info()) -def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C float row to python row + +def taos_stmt_errstr(stmt): + # type: (ctypes.c_void_p) -> str + """Get error message from stetement query + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + err = c_char_p(_libtaos.taos_stmt_errstr(stmt)) + if err: + return err.value.decode("utf-8") + +try: + _libtaos.taos_stmt_set_tbname.restype = c_int + _libtaos.taos_stmt_set_tbname.argstype = (c_void_p, c_char_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname" % taos_get_client_info()) -def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C double row to python row + +def taos_stmt_set_tbname(stmt, name): + # type: (ctypes.c_void_p, str) -> None + """Set table name of a statement query if exists. + @stmt: c_void_p TAOS_STMT* """ - return [None if math.isnan(ele) else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_set_tbname(stmt, c_char_p(name.encode("utf-8"))) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_set_tbname_tags.restype = c_int + _libtaos.taos_stmt_set_tbname_tags.argstype = (c_void_p, c_char_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_set_tbname_tags" % taos_get_client_info()) -def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row + +def taos_stmt_set_tbname_tags(stmt, name, tags): + # type: (c_void_p, str, c_void_p) -> None + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* """ - assert(nbytes is not None) - return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( - 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + res = _libtaos.taos_stmt_set_tbname_tags(stmt, ctypes.c_char_p(name.encode("utf-8")), tags) + + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row +_libtaos.taos_stmt_is_insert.restype = c_int +_libtaos.taos_stmt_is_insert.argstype = (c_void_p, POINTER(c_int)) + + +def taos_stmt_is_insert(stmt): + # type: (ctypes.c_void_p) -> bool + """Set table name with tags bind params. + @stmt: c_void_p TAOS_STMT* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - if num_of_rows >= 0: - tmpstr = ctypes.c_char_p(data) - res.append(tmpstr.value.decode()) - else: - res.append((ctypes.cast(data + nbytes * i, - ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) - except ValueError: - res.append(None) + is_insert = ctypes.c_int() + res = _libtaos.taos_stmt_is_insert(stmt, ctypes.byref(is_insert)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return is_insert == 0 + - return res +_libtaos.taos_stmt_num_params.restype = c_int +_libtaos.taos_stmt_num_params.argstype = (c_void_p, POINTER(c_int)) -def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C binary row to python row +def taos_stmt_num_params(stmt): + # type: (ctypes.c_void_p) -> int + """Params number of the current statement query. + @stmt: TAOS_STMT* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - rbyte = ctypes.cast( - data + nbytes * i, - ctypes.POINTER( - ctypes.c_short))[ - :1].pop() - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()[0:rbyte]) - except ValueError: - res.append(None) - return res - - -def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): - """Function to convert C nchar row to python row + num_params = ctypes.c_int() + res = _libtaos.taos_stmt_num_params(stmt, ctypes.byref(num_params)) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + return num_params.value + + +_libtaos.taos_stmt_bind_param.restype = c_int +_libtaos.taos_stmt_bind_param.argstype = (c_void_p, c_void_p) + + +def taos_stmt_bind_param(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* """ - assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): - try: - tmpstr = ctypes.c_char_p(data + nbytes * i + 2) - res.append(tmpstr.value.decode()) - except ValueError: - res.append(None) - return res - - -_CONVERT_FUNC = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -_CONVERT_FUNC_BLOCK = { - FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT: _crow_tinyint_to_python, - FieldType.C_SMALLINT: _crow_smallint_to_python, - FieldType.C_INT: _crow_int_to_python, - FieldType.C_BIGINT: _crow_bigint_to_python, - FieldType.C_FLOAT: _crow_float_to_python, - FieldType.C_DOUBLE: _crow_double_to_python, - FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP: _crow_timestamp_to_python, - FieldType.C_NCHAR: _crow_nchar_to_python_block, - FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, - FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, - FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, - FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python -} - -# Corresponding TAOS_FIELD structure in C - - -class TaosField(ctypes.Structure): - _fields_ = [('name', ctypes.c_char * 65), - ('type', ctypes.c_char), - ('bytes', ctypes.c_short)] + # ptr = ctypes.cast(bind, POINTER(TaosBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -# C interface class +try: + _libtaos.taos_stmt_bind_param_batch.restype = c_int + _libtaos.taos_stmt_bind_param_batch.argstype = (c_void_p, c_void_p) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_param_batch" % taos_get_client_info()) -def _load_taos_linux(): - return ctypes.CDLL('libtaos.so') +def taos_stmt_bind_param_batch(stmt, bind): + # type: (ctypes.c_void_p, Array[TaosMultiBind]) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_BIND* + """ + # ptr = ctypes.cast(bind, POINTER(TaosMultiBind)) + # ptr = pointer(bind) + res = _libtaos.taos_stmt_bind_param_batch(stmt, bind) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + +try: + _libtaos.taos_stmt_bind_single_param_batch.restype = c_int + _libtaos.taos_stmt_bind_single_param_batch.argstype = (c_void_p, c_void_p, c_int) +except AttributeError: + print("WARNING: libtaos(%s) does not support taos_stmt_bind_single_param_batch" % taos_get_client_info()) + + +def taos_stmt_bind_single_param_batch(stmt, bind, col): + # type: (ctypes.c_void_p, Array[TaosMultiBind], c_int) -> None + """Bind params in the statement query. + @stmt: TAOS_STMT* + @bind: TAOS_MULTI_BIND* + @col: column index + """ + res = _libtaos.taos_stmt_bind_single_param_batch(stmt, bind, col) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos_darwin(): - return ctypes.CDLL('libtaos.dylib') +_libtaos.taos_stmt_add_batch.restype = c_int +_libtaos.taos_stmt_add_batch.argstype = (c_void_p,) -def _load_taos_windows(): - return ctypes.windll.LoadLibrary('taos') +def taos_stmt_add_batch(stmt): + # type: (ctypes.c_void_p) -> None + """Add current params into batch + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_add_batch(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) -def _load_taos(): - load_func = { - 'Linux': _load_taos_linux, - 'Darwin': _load_taos_darwin, - 'Windows': _load_taos_windows, - } - try: - return load_func[platform.system()]() - except: - raise InterfaceError('unsupported platform or failed to load taos client library') +_libtaos.taos_stmt_execute.restype = c_int +_libtaos.taos_stmt_execute.argstype = (c_void_p,) -class CTaosInterface(object): - libtaos = _load_taos() - - libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) - libtaos.taos_init.restype = None - libtaos.taos_connect.restype = ctypes.c_void_p - # libtaos.taos_use_result.restype = ctypes.c_void_p - libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) - libtaos.taos_errstr.restype = ctypes.c_char_p - libtaos.taos_subscribe.restype = ctypes.c_void_p - libtaos.taos_consume.restype = ctypes.c_void_p - libtaos.taos_fetch_lengths.restype = ctypes.c_void_p - libtaos.taos_free_result.restype = None - libtaos.taos_errno.restype = ctypes.c_int - libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +def taos_stmt_execute(stmt): + # type: (ctypes.c_void_p) -> None + """Execute a statement query + @stmt: TAOS_STMT* + """ + res = _libtaos.taos_stmt_execute(stmt) + if res != 0: + raise StatementError(msg=taos_stmt_errstr(stmt), errno=res) + + +_libtaos.taos_stmt_use_result.restype = c_void_p +_libtaos.taos_stmt_use_result.argstype = (c_void_p,) + + +def taos_stmt_use_result(stmt): + # type: (ctypes.c_void_p) -> None + """Get result of the statement. + @stmt: TAOS_STMT* + """ + result = c_void_p(_libtaos.taos_stmt_use_result(stmt)) + if result == None: + raise StatementError(taos_stmt_errstr(stmt)) + return result + +try: + _libtaos.taos_insert_lines.restype = c_int + _libtaos.taos_insert_lines.argstype = c_void_p, c_void_p, c_int +except AttributeError: + print("WARNING: libtaos(%s) does not support insert_lines" % taos_get_client_info()) + + + +def taos_insert_lines(connection, lines): + # type: (c_void_p, list[str] | tuple(str)) -> None + num_of_lines = len(lines) + lines = (c_char_p(line.encode("utf-8")) for line in lines) + lines_type = ctypes.c_char_p * num_of_lines + p_lines = lines_type(*lines) + errno = _libtaos.taos_insert_lines(connection, p_lines, num_of_lines) + if errno != 0: + raise LinesError("insert lines error", errno) + + +class CTaosInterface(object): def __init__(self, config=None): - ''' + """ Function to initialize the class @host : str, hostname to connect @user : str, username to connect to server @@ -298,304 +842,46 @@ class CTaosInterface(object): @config : str, config directory @rtype : None - ''' + """ if config is None: self._config = ctypes.c_char_p(None) else: try: - self._config = ctypes.c_char_p(config.encode('utf-8')) + self._config = ctypes.c_char_p(config.encode("utf-8")) except AttributeError: raise AttributeError("config is expected as a str") if config is not None: - CTaosInterface.libtaos.taos_options(3, self._config) + taos_options(3, self._config) - CTaosInterface.libtaos.taos_init() + taos_init() @property def config(self): - """ Get current config - """ + """Get current config""" return self._config - def connect( - self, - host=None, - user="root", - password="taosdata", - db=None, - port=0): - ''' + def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + """ Function to connect to server @rtype: c_void_p, TDengine handle - ''' - # host - try: - _host = ctypes.c_char_p(host.encode( - "utf-8")) if host is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("host is expected as a str") - - # user - try: - _user = ctypes.c_char_p(user.encode("utf-8")) - except AttributeError: - raise AttributeError("user is expected as a str") - - # password - try: - _password = ctypes.c_char_p(password.encode("utf-8")) - except AttributeError: - raise AttributeError("password is expected as a str") - - # db - try: - _db = ctypes.c_char_p( - db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) - except AttributeError: - raise AttributeError("db is expected as a str") - - # port - try: - _port = ctypes.c_int(port) - except TypeError: - raise TypeError("port is expected as an int") - - connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( - _host, _user, _password, _db, _port)) - - if connection.value is None: - print('connect to TDengine failed') - raise ConnectionError("connect to TDengine failed") - # sys.exit(1) - # else: - # print('connect to TDengine success') - - return connection - - @staticmethod - def close(connection): - '''Close the TDengine handle - ''' - CTaosInterface.libtaos.taos_close(connection) - # print('connection is closed') - - @staticmethod - def query(connection, sql): - '''Run SQL - - @sql: str, sql string to run - - @rtype: 0 on success and -1 on failure - ''' - try: - return CTaosInterface.libtaos.taos_query( - connection, ctypes.c_char_p(sql.encode('utf-8'))) - except AttributeError: - raise AttributeError("sql is expected as a string") - # finally: - # CTaosInterface.libtaos.close(connection) - - @staticmethod - def affectedRows(result): - """The affected rows after runing query - """ - return CTaosInterface.libtaos.taos_affected_rows(result) - - @staticmethod - def insertLines(connection, lines): - ''' - insert through lines protocol - @lines: list of str - @rtype: tsdb error codes - ''' - numLines = len(lines) - c_lines_type = ctypes.c_char_p*numLines - c_lines = c_lines_type() - for i in range(numLines): - c_lines[i] = ctypes.c_char_p(lines[i].encode('utf-8')) - return CTaosInterface.libtaos.taos_insert_lines(connection, c_lines, ctypes.c_int(numLines)) - - @staticmethod - def subscribe(connection, restart, topic, sql, interval): - """Create a subscription - @restart boolean, - @sql string, sql statement for data query, must be a 'select' statement. - @topic string, name of this subscription - """ - return ctypes.c_void_p(CTaosInterface.libtaos.taos_subscribe( - connection, - 1 if restart else 0, - ctypes.c_char_p(topic.encode('utf-8')), - ctypes.c_char_p(sql.encode('utf-8')), - None, - None, - interval)) - - @staticmethod - def consume(sub): - """Consume data of a subscription - """ - result = ctypes.c_void_p(CTaosInterface.libtaos.taos_consume(sub)) - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.libtaos.taos_num_fields(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - return result, fields - - @staticmethod - def unsubscribe(sub, keepProgress): - """Cancel a subscription - """ - CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) - - @staticmethod - def useResult(result): - '''Use result after calling self.query - ''' - fields = [] - pfields = CTaosInterface.fetchFields(result) - for i in range(CTaosInterface.fieldsCount(result)): - fields.append({'name': pfields[i].name.decode('utf-8'), - 'bytes': pfields[i].bytes, - 'type': ord(pfields[i].type)}) - - return fields - - @staticmethod - def fetchBlock(result, fields): - pblock = ctypes.c_void_p(0) - num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - result, ctypes.byref(pblock)) - if num_of_rows == 0: - return None, 0 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: - raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - - return blocks, abs(num_of_rows) - - @staticmethod - def fetchRow(result, fields): - pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock: - num_of_rows = 1 - precision = CTaosInterface.libtaos.taos_result_precision(result) - blocks = [None] * len(fields) - fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ - ele for ele in ctypes.cast( - fieldL, ctypes.POINTER( - ctypes.c_int))[ - :len(fields)]] - for i in range(len(fields)): - data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError( - "Invalid data type returned from database") - if data is None: - blocks[i] = [None] - else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']]( - data, num_of_rows, fieldLen[i], precision) - else: - return None, 0 - return blocks, abs(num_of_rows) - - @staticmethod - def freeResult(result): - CTaosInterface.libtaos.taos_free_result(result) - result.value = None - - @staticmethod - def fieldsCount(result): - return CTaosInterface.libtaos.taos_field_count(result) - - @staticmethod - def fetchFields(result): - return CTaosInterface.libtaos.taos_fetch_fields(result) - - # @staticmethod - # def fetchRow(result, fields): - # l = [] - # row = CTaosInterface.libtaos.taos_fetch_row(result) - # if not row: - # return None - - # for i in range(len(fields)): - # l.append(CTaosInterface.getDataValue( - # row[i], fields[i]['type'], fields[i]['bytes'])) - - # return tuple(l) - - # @staticmethod - # def getDataValue(data, dtype, byte): - # ''' - # ''' - # if not data: - # return None - - # if (dtype == CTaosInterface.TSDB_DATA_TYPE_BOOL): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TINYINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_SMALLINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): - # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] - # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): - # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') - - @staticmethod - def errno(result): - """Return the error number. - """ - return CTaosInterface.libtaos.taos_errno(result) - - @staticmethod - def errStr(result): - """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') + return taos_connect(host, user, password, db, port) -if __name__ == '__main__': +if __name__ == "__main__": cinter = CTaosInterface() conn = cinter.connect() - result = cinter.query(conn, 'show databases') + result = cinter.query(conn, "show databases") - print('Query Affected rows: {}'.format(cinter.affectedRows(result))) + print("Query Affected rows: {}".format(cinter.affected_rows(result))) - fields = CTaosInterface.useResult(result) + fields = taos_fetch_fields_raw(result) - data, num_of_rows = CTaosInterface.fetchBlock(result, fields) + data, num_of_rows = taos_fetch_block(result, fields) print(data) - cinter.freeResult(result) + cinter.free_result(result) cinter.close(conn) diff --git a/src/connector/python/taos/connection.py b/src/connector/python/taos/connection.py index 88d06cd7186018788aeb25c982fc205441193cb8..7857c8c706dbe27fd9440e6bf2eb698b6822650e 100644 --- a/src/connector/python/taos/connection.py +++ b/src/connector/python/taos/connection.py @@ -1,11 +1,15 @@ -from .cursor import TDengineCursor -from .subscription import TDengineSubscription -from .cinterface import CTaosInterface +# encoding:UTF-8 +from types import FunctionType +from .cinterface import * +from .cursor import TaosCursor +from .subscription import TaosSubscription +from .statement import TaosStmt +from .stream import TaosStream +from .result import * -class TDengineConnection(object): - """ TDengine connection object - """ +class TaosConnection(object): + """TDengine connection object""" def __init__(self, *args, **kwargs): self._conn = None @@ -21,63 +25,130 @@ class TDengineConnection(object): def config(self, **kwargs): # host - if 'host' in kwargs: - self._host = kwargs['host'] + if "host" in kwargs: + self._host = kwargs["host"] # user - if 'user' in kwargs: - self._user = kwargs['user'] + if "user" in kwargs: + self._user = kwargs["user"] # password - if 'password' in kwargs: - self._password = kwargs['password'] + if "password" in kwargs: + self._password = kwargs["password"] # database - if 'database' in kwargs: - self._database = kwargs['database'] + if "database" in kwargs: + self._database = kwargs["database"] # port - if 'port' in kwargs: - self._port = kwargs['port'] + if "port" in kwargs: + self._port = kwargs["port"] # config - if 'config' in kwargs: - self._config = kwargs['config'] + if "config" in kwargs: + self._config = kwargs["config"] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect( - self._host, - self._user, - self._password, - self._database, - self._port) + self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) def close(self): - """Close current connection. - """ - return CTaosInterface.close(self._conn) - - def subscribe(self, restart, topic, sql, interval): - """Create a subscription. - """ + """Close current connection.""" + if self._conn: + taos_close(self._conn) + self._conn = None + + @property + def client_info(self): + # type: () -> str + return taos_get_client_info() + + @property + def server_info(self): + # type: () -> str + return taos_get_server_info(self._conn) + + def select_db(self, database): + # type: (str) -> None + taos_select_db(self._conn, database) + + def execute(self, sql): + # type: (str) -> None + """Simplely execute sql ignoring the results""" + res = taos_query(self._conn, sql) + taos_free_result(res) + + def query(self, sql): + # type: (str) -> TaosResult + result = taos_query(self._conn, sql) + return TaosResult(result, True, self) + + def query_a(self, sql, callback, param): + # type: (str, async_query_callback_type, c_void_p) -> None + """Asynchronously query a sql with callback function""" + taos_query_a(self._conn, sql, callback, param) + + def subscribe(self, restart, topic, sql, interval, callback=None, param=None): + # type: (bool, str, str, int, subscribe_callback_type, c_void_p) -> TaosSubscription + """Create a subscription.""" if self._conn is None: return None - sub = CTaosInterface.subscribe( - self._conn, restart, topic, sql, interval) - return TDengineSubscription(sub) + sub = taos_subscribe(self._conn, restart, topic, sql, interval, callback, param) + return TaosSubscription(sub, callback != None) - def insertLines(self, lines): - """ - insert lines through line protocol - """ + def statement(self, sql=None): + # type: (str | None) -> TaosStmt if self._conn is None: return None - return CTaosInterface.insertLines(self._conn, lines) - - def cursor(self): - """Return a new Cursor object using the connection. + stmt = taos_stmt_init(self._conn) + if sql != None: + taos_stmt_prepare(stmt, sql) + + return TaosStmt(stmt) + + def load_table_info(self, tables): + # type: (str) -> None + taos_load_table_info(self._conn, tables) + + def stream(self, sql, callback, stime=0, param=None, callback2=None): + # type: (str, Callable[[Any, TaosResult, TaosRows], None], int, Any, c_void_p) -> TaosStream + # cb = cast(callback, stream_callback_type) + # ref = byref(cb) + + stream = taos_open_stream(self._conn, sql, callback, stime, param, callback2) + return TaosStream(stream) + + def insert_lines(self, lines): + # type: (list[str]) -> None + """Line protocol and schemaless support + + ## Example + + ```python + import taos + conn = taos.connect() + conn.exec("drop database if exists test") + conn.select_db("test") + lines = [ + 'ste,t2=5,t3=L"ste" c1=true,c2=4,c3="string" 1626056811855516532', + ] + conn.insert_lines(lines) + ``` + + ## Exception + + ```python + try: + conn.insert_lines(lines) + except SchemalessError as err: + print(err) + ``` """ - return TDengineCursor(self) + return taos_insert_lines(self._conn, lines) + + def cursor(self): + # type: () -> TaosCursor + """Return a new Cursor object using the connection.""" + return TaosCursor(self) def commit(self): """Commit any pending transaction to the database. @@ -87,17 +158,18 @@ class TDengineConnection(object): pass def rollback(self): - """Void functionality - """ + """Void functionality""" pass def clear_result_set(self): - """Clear unused result set on this connection. - """ + """Clear unused result set on this connection.""" pass + def __del__(self): + self.close() + if __name__ == "__main__": - conn = TDengineConnection(host='192.168.1.107') + conn = TaosConnection() conn.close() print("Hello world") diff --git a/src/connector/python/taos/constants.py b/src/connector/python/taos/constants.py index 85689b02db76b0032558b983d8ae8d9297229c42..8ad5b69fc099718fa4f4b8c08cf689b17663eae0 100644 --- a/src/connector/python/taos/constants.py +++ b/src/connector/python/taos/constants.py @@ -1,12 +1,14 @@ +# encoding:UTF-8 + """Constants in TDengine python """ -from .dbapi import * +import ctypes, struct class FieldType(object): - """TDengine Field Types - """ + """TDengine Field Types""" + # type_code C_NULL = 0 C_BOOL = 1 @@ -34,9 +36,9 @@ class FieldType(object): C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 C_BIGINT_UNSIGNED_NULL = 18446744073709551615 - C_FLOAT_NULL = float('nan') - C_DOUBLE_NULL = float('nan') - C_BINARY_NULL = bytearray([int('0xff', 16)]) + C_FLOAT_NULL = ctypes.c_float(struct.unpack(" name (mondatory) - > type_code (mondatory) + > name (mandatory) + > type_code (mandatory) > display_size > internal_size > precision @@ -55,8 +55,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow( - self._result, self._fields) + block, self._block_rows = taos_fetch_row(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -69,20 +68,17 @@ class TDengineCursor(object): @property def description(self): - """Return the description of the object. - """ + """Return the description of the object.""" return self._description @property def rowcount(self): - """Return the rowcount of the object - """ + """Return the rowcount of the object""" return self._rowcount @property def affected_rows(self): - """Return the rowcount of insertion - """ + """Return the rowcount of insertion""" return self._affected_rows def callproc(self, procname, *args): @@ -96,8 +92,7 @@ class TDengineCursor(object): self._logfile = logfile def close(self): - """Close the cursor. - """ + """Close the cursor.""" if self._connection is None: return False @@ -107,8 +102,7 @@ class TDengineCursor(object): return True def execute(self, operation, params=None): - """Prepare and execute a database operation (query or command). - """ + """Prepare and execute a database operation (query or command).""" if not operation: return None @@ -124,104 +118,91 @@ class TDengineCursor(object): # global querySeqNum # querySeqNum += 1 - # localSeqNum = querySeqNum # avoid raice condition + # localSeqNum = querySeqNum # avoid race condition # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) - self._result = CTaosInterface.query(self._connection._conn, stmt) + self._result = taos_query(self._connection._conn, stmt) # print(" << Query ({}) Exec Done".format(localSeqNum)) - if (self._logfile): + if self._logfile: with open(self._logfile, "a") as logfile: logfile.write("%s;\n" % operation) - errno = CTaosInterface.libtaos.taos_errno(self._result) - if errno == 0: - if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows( - self._result) - return CTaosInterface.affectedRows(self._result) - else: - self._fields = CTaosInterface.useResult( - self._result) - return self._handle_result() + if taos_field_count(self._result) == 0: + affected_rows = taos_affected_rows(self._result) + self._affected_rows += affected_rows + return affected_rows else: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + self._fields = taos_fetch_fields(self._result) + return self._handle_result() def executemany(self, operation, seq_of_parameters): - """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. - """ + """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters.""" pass def fetchone(self): - """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. - """ + """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available.""" pass def fetchmany(self): pass def istype(self, col, dataType): - if (dataType.upper() == "BOOL"): - if (self._description[col][1] == FieldType.C_BOOL): + if dataType.upper() == "BOOL": + if self._description[col][1] == FieldType.C_BOOL: return True - if (dataType.upper() == "TINYINT"): - if (self._description[col][1] == FieldType.C_TINYINT): + if dataType.upper() == "TINYINT": + if self._description[col][1] == FieldType.C_TINYINT: return True - if (dataType.upper() == "TINYINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + if dataType.upper() == "TINYINT UNSIGNED": + if self._description[col][1] == FieldType.C_TINYINT_UNSIGNED: return True - if (dataType.upper() == "SMALLINT"): - if (self._description[col][1] == FieldType.C_SMALLINT): + if dataType.upper() == "SMALLINT": + if self._description[col][1] == FieldType.C_SMALLINT: return True - if (dataType.upper() == "SMALLINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + if dataType.upper() == "SMALLINT UNSIGNED": + if self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED: return True - if (dataType.upper() == "INT"): - if (self._description[col][1] == FieldType.C_INT): + if dataType.upper() == "INT": + if self._description[col][1] == FieldType.C_INT: return True - if (dataType.upper() == "INT UNSIGNED"): - if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + if dataType.upper() == "INT UNSIGNED": + if self._description[col][1] == FieldType.C_INT_UNSIGNED: return True - if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_BIGINT): + if dataType.upper() == "BIGINT": + if self._description[col][1] == FieldType.C_BIGINT: return True - if (dataType.upper() == "BIGINT UNSIGNED"): - if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): + if dataType.upper() == "BIGINT UNSIGNED": + if self._description[col][1] == FieldType.C_BIGINT_UNSIGNED: return True - if (dataType.upper() == "FLOAT"): - if (self._description[col][1] == FieldType.C_FLOAT): + if dataType.upper() == "FLOAT": + if self._description[col][1] == FieldType.C_FLOAT: return True - if (dataType.upper() == "DOUBLE"): - if (self._description[col][1] == FieldType.C_DOUBLE): + if dataType.upper() == "DOUBLE": + if self._description[col][1] == FieldType.C_DOUBLE: return True - if (dataType.upper() == "BINARY"): - if (self._description[col][1] == FieldType.C_BINARY): + if dataType.upper() == "BINARY": + if self._description[col][1] == FieldType.C_BINARY: return True - if (dataType.upper() == "TIMESTAMP"): - if (self._description[col][1] == FieldType.C_TIMESTAMP): + if dataType.upper() == "TIMESTAMP": + if self._description[col][1] == FieldType.C_TIMESTAMP: return True - if (dataType.upper() == "NCHAR"): - if (self._description[col][1] == FieldType.C_NCHAR): + if dataType.upper() == "NCHAR": + if self._description[col][1] == FieldType.C_NCHAR: return True return False def fetchall_row(self): - """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. - """ + """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.""" if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_row(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -230,19 +211,16 @@ class TDengineCursor(object): return list(map(tuple, zip(*buffer))) def fetchall(self): - if self._result is None or self._fields is None: + if self._result is None: raise OperationalError("Invalid use of fetchall") - - buffer = [[] for i in range(len(self._fields))] + fields = self._fields if self._fields is not None else taos_fetch_fields(self._result) + buffer = [[] for i in range(len(fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock( - self._result, self._fields) - errno = CTaosInterface.libtaos.taos_errno(self._result) + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) if errno != 0: - raise ProgrammingError( - CTaosInterface.errStr( - self._result), errno) + raise ProgrammingError(taos_errstr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -250,9 +228,12 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def stop_query(self): + if self._result != None: + taos_stop_query(self._result) + def nextset(self): - """ - """ + """ """ pass def setinputsize(self, sizes): @@ -262,12 +243,11 @@ class TDengineCursor(object): pass def _reset_result(self): - """Reset the result to unused version. - """ + """Reset the result to unused version.""" self._description = [] self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + taos_free_result(self._result) self._result = None self._fields = None self._block = None @@ -276,11 +256,12 @@ class TDengineCursor(object): self._affected_rows = 0 def _handle_result(self): - """Handle the return result from query. - """ + """Handle the return result from query.""" self._description = [] for ele in self._fields: - self._description.append( - (ele['name'], ele['type'], None, None, None, None, False)) + self._description.append((ele["name"], ele["type"], None, None, None, None, False)) return self._result + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/dbapi.py b/src/connector/python/taos/dbapi.py deleted file mode 100644 index 594681ada953abf388e503c23199043cf686e1a3..0000000000000000000000000000000000000000 --- a/src/connector/python/taos/dbapi.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Type Objects and Constructors. -""" - -import time -import datetime - - -class DBAPITypeObject(object): - def __init__(self, *values): - self.values = values - - def __com__(self, other): - if other in self.values: - return 0 - if other < self.values: - return 1 - else: - return -1 - - -Date = datetime.date -Time = datetime.time -Timestamp = datetime.datetime - - -def DataFromTicks(ticks): - return Date(*time.localtime(ticks)[:3]) - - -def TimeFromTicks(ticks): - return Time(*time.localtime(ticks)[3:6]) - - -def TimestampFromTicks(ticks): - return Timestamp(*time.localtime(ticks)[:6]) - - -Binary = bytes - -# STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) -# BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) -# NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) -# DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() diff --git a/src/connector/python/taos/error.py b/src/connector/python/taos/error.py index c584badce8320cd35dc81e8f6b613c56163b1a29..a30adbb162f1c194bdfcf4cca5c43f01107a9776 100644 --- a/src/connector/python/taos/error.py +++ b/src/connector/python/taos/error.py @@ -1,66 +1,86 @@ +# encoding:UTF-8 """Python exceptions """ class Error(Exception): - def __init__(self, msg=None, errno=None): + def __init__(self, msg=None, errno=0xffff): self.msg = msg - self._full_msg = self.msg self.errno = errno + self._full_msg = "[0x%04x]: %s" % (self.errno & 0xffff, self.msg) def __str__(self): return self._full_msg class Warning(Exception): - """Exception raised for important warnings like data truncations while inserting. - """ + """Exception raised for important warnings like data truncations while inserting.""" + pass class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. - """ + """Exception raised for errors that are related to the database interface rather than the database itself.""" + pass class DatabaseError(Error): - """Exception raised for errors that are related to the database. - """ + """Exception raised for errors that are related to the database.""" + pass +class ConnectionError(Error): + """Exceptin raised for connection failed""" + pass class DataError(DatabaseError): - """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. - """ + """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range.""" + pass class OperationalError(DatabaseError): - """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer - """ + """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer""" + pass class IntegrityError(DatabaseError): - """Exception raised when the relational integrity of the database is affected. - """ + """Exception raised when the relational integrity of the database is affected.""" + pass class InternalError(DatabaseError): - """Exception raised when the database encounters an internal error. - """ + """Exception raised when the database encounters an internal error.""" + pass class ProgrammingError(DatabaseError): - """Exception raised for programming errors. - """ + """Exception raised for programming errors.""" + pass class NotSupportedError(DatabaseError): - """Exception raised in case a method or database API was used which is not supported by the database,. - """ + """Exception raised in case a method or database API was used which is not supported by the database,.""" + pass + + +class StatementError(DatabaseError): + """Exception raised in STMT API.""" + + pass + +class ResultError(DatabaseError): + """Result related APIs.""" + + pass + +class LinesError(DatabaseError): + """taos_insert_lines errors.""" + + pass \ No newline at end of file diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py new file mode 100644 index 0000000000000000000000000000000000000000..445cd8afdba6f2512c73be95c9b0dbd8dc00da8a --- /dev/null +++ b/src/connector/python/taos/field.py @@ -0,0 +1,302 @@ +# encoding:UTF-8 +import ctypes +import math +import datetime +from ctypes import * + +from .constants import FieldType +from .error import * + +_datetime_epoch = datetime.datetime.fromtimestamp(0) + +def _convert_millisecond_to_datetime(milli): + return _datetime_epoch + datetime.timedelta(seconds=milli / 1000.0) + + +def _convert_microsecond_to_datetime(micro): + return _datetime_epoch + datetime.timedelta(seconds=micro / 1000000.0) + + +def _convert_nanosecond_to_datetime(nanosec): + return nanosec + + +def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + _timestamp_converter = _convert_millisecond_to_datetime + if precision == FieldType.C_TIMESTAMP_MILLI: + _timestamp_converter = _convert_millisecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_MICRO: + _timestamp_converter = _convert_microsecond_to_datetime + elif precision == FieldType.C_TIMESTAMP_NANO: + _timestamp_converter = _convert_nanosecond_to_datetime + else: + raise DatabaseError("Unknown precision returned from database") + + return [ + None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bool row to python row""" + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[: abs(num_of_rows)] + ] + + +def _crow_tinyint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C tinyint row to python row""" + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ubyte))[: abs(num_of_rows)] + ] + + +def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[: abs(num_of_rows)] + ] + + +def _crow_smallint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C smallint row to python row""" + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_ushort))[: abs(num_of_rows)] + ] + + +def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[: abs(num_of_rows)] + ] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C int row to python row""" + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[: abs(num_of_rows)] + ] + + +def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[: abs(num_of_rows)] + ] + + +def _crow_bigint_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C bigint row to python row""" + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_uint64))[: abs(num_of_rows)] + ] + + +def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C float row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[: abs(num_of_rows)] + ] + + +def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C double row to python row""" + return [ + None if math.isnan(ele) else ele + for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[: abs(num_of_rows)] + ] + + +def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + return [ + None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode("utf-8") + for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[: abs(num_of_rows)] + ] + + +def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + if num_of_rows >= 0: + tmpstr = ctypes.c_char_p(data) + res.append(tmpstr.value.decode()) + else: + res.append( + ( + ctypes.cast( + data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4)), + ) + )[0].value + ) + except ValueError: + res.append(None) + + return res + + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C binary row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast(data + nbytes * i, ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + return res + + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN): + """Function to convert C nchar row to python row""" + assert nbytes is not None + res = [] + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) + except ValueError: + res.append(None) + return res + + +CONVERT_FUNC = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, +} + +# Corresponding TAOS_FIELD structure in C + + +class TaosField(ctypes.Structure): + _fields_ = [ + ("_name", ctypes.c_char * 65), + ("_type", ctypes.c_uint8), + ("_bytes", ctypes.c_uint16), + ] + + @property + def name(self): + return self._name.decode("utf-8") + + @property + def length(self): + """alias to self.bytes""" + return self._bytes + + @property + def bytes(self): + return self._bytes + + @property + def type(self): + return self._type + + def __dict__(self): + return {"name": self.name, "type": self.type, "bytes": self.length} + + def __str__(self): + return "{name: %s, type: %d, bytes: %d}" % (self.name, self.type, self.length) + + def __getitem__(self, item): + return getattr(self, item) + + +class TaosFields(object): + def __init__(self, fields, count): + if isinstance(fields, c_void_p): + self._fields = cast(fields, POINTER(TaosField)) + if isinstance(fields, POINTER(TaosField)): + self._fields = fields + self._count = count + self._iter = 0 + + def as_ptr(self): + return self._fields + + @property + def count(self): + return self._count + + @property + def fields(self): + return self._fields + + def __next__(self): + return self._next_field() + + def next(self): + return self._next_field() + + def _next_field(self): + if self._iter < self.count: + field = self._fields[self._iter] + self._iter += 1 + return field + else: + raise StopIteration + + def __getitem__(self, item): + return self._fields[item] + + def __iter__(self): + return self + + def __len__(self): + return self.count diff --git a/src/connector/python/taos/precision.py b/src/connector/python/taos/precision.py new file mode 100644 index 0000000000000000000000000000000000000000..d67da592cce6d2121ec8f2eed78a30d6fa0c446b --- /dev/null +++ b/src/connector/python/taos/precision.py @@ -0,0 +1,12 @@ +class PrecisionEnum(object): + """Precision enums""" + + Milliseconds = 0 + Microseconds = 1 + Nanoseconds = 2 + + +class PrecisionError(Exception): + """Python datetime does not support nanoseconds error""" + + pass diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py new file mode 100644 index 0000000000000000000000000000000000000000..81151733615d1b7fdc3318b6e53888ae39d32b14 --- /dev/null +++ b/src/connector/python/taos/result.py @@ -0,0 +1,245 @@ +from .cinterface import * + +# from .connection import TaosConnection +from .error import * + + +class TaosResult(object): + """TDengine result interface""" + + def __init__(self, result, close_after=False, conn=None): + # type: (c_void_p, bool, TaosConnection) -> TaosResult + # to make the __del__ order right + self._conn = conn + self._close_after = close_after + self._result = result + self._fields = None + self._field_count = None + self._precision = None + + self._block = None + self._block_length = None + self._row_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + # fetch next row + return self._next_row() + + def _next_row(self): + if self._result is None or self.fields is None: + raise OperationalError("Invalid use of fetch iterator") + + if self._block == None or self._block_iter >= self._block_length: + self._block, self._block_length = self.fetch_block() + self._block_iter = 0 + # self._row_count += self._block_length + + raw = self._block[self._block_iter] + self._block_iter += 1 + return raw + + @property + def fields(self): + """fields definitions of the current result""" + if self._result is None: + raise ResultError("no result object setted") + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + + return self._fields + + @property + def field_count(self): + """Field count of the current result, eq to taos_field_count(result)""" + return self.fields.count + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._row_count + + @property + def precision(self): + if self._precision == None: + self._precision = taos_result_precision(self._result) + return self._precision + + @property + def affected_rows(self): + return taos_affected_rows(self._result) + + # @property + def field_lengths(self): + return taos_fetch_lengths(self._result, self.field_count) + + def rows_iter(self, num_of_rows=None): + return TaosRows(self, num_of_rows) + + def blocks_iter(self): + return TaosBlocks(self) + + def fetch_block(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + + block, length = taos_fetch_block_raw(self._result) + if length == 0: + raise StopIteration + precision = self.precision + field_count = self.field_count + fields = self.fields + blocks = [None] * field_count + lengths = self.field_lengths() + for i in range(field_count): + data = ctypes.cast(block, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = CONVERT_FUNC_BLOCK[fields[i].type](data, length, lengths[i], precision) + + return list(map(tuple, zip(*blocks))), length + + def fetch_all(self): + if self._result is None: + raise OperationalError("Invalid use of fetchall") + + if self._fields == None: + self._fields = taos_fetch_fields(self._result) + buffer = [[] for i in range(len(self._fields))] + self._row_count = 0 + while True: + block, num_of_fields = taos_fetch_block(self._result, self._fields) + errno = taos_errno(self._result) + if errno != 0: + raise ProgrammingError(taos_errstr(self._result), errno) + if num_of_fields == 0: + break + self._row_count += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetch_rows_a(self, callback, param): + taos_fetch_rows_a(self._result, callback, param) + + def stop_query(self): + return taos_stop_query(self._result) + + def errno(self): + """**DO NOT** use this directly unless you know what you are doing""" + return taos_errno(self._result) + + def errstr(self): + return taos_errstr(self._result) + + def check_error(self, errno=None, close=True): + if errno == None: + errno = self.errno() + if errno != 0: + msg = self.errstr() + self.close() + raise OperationalError(msg, errno) + + def close(self): + """free result object.""" + if self._result != None and self._close_after: + taos_free_result(self._result) + self._result = None + self._fields = None + self._field_count = None + self._field_lengths = None + + def __del__(self): + self.close() + + +class TaosRows: + """TDengine result rows iterator""" + + def __init__(self, result, num_of_rows=None): + self._result = result + self._num_of_rows = num_of_rows + + def __iter__(self): + return self + + def __next__(self): + return self._next_row() + + def next(self): + return self._next_row() + + def _next_row(self): + if self._result is None: + raise OperationalError("Invalid use of fetch iterator") + if self._num_of_rows != None and self._num_of_rows <= self._result._row_count: + raise StopIteration + + row = taos_fetch_row_raw(self._result._result) + if not row: + raise StopIteration + self._result._row_count += 1 + return TaosRow(self._result, row) + + @property + def row_count(self): + """Return the rowcount of the object""" + return self._result._row_count + + +class TaosRow: + def __init__(self, result, row): + self._result = result + self._row = row + + def __str__(self): + return taos_print_row(self._row, self._result.fields, self._result.field_count) + + def __call__(self): + return self.as_tuple() + + def _astuple(self): + return self.as_tuple() + + def __iter__(self): + return self.as_tuple() + + def as_ptr(self): + return self._row + + def as_tuple(self): + precision = self._result.precision + field_count = self._result.field_count + blocks = [None] * field_count + fields = self._result.fields + field_lens = self._result.field_lengths() + for i in range(field_count): + data = ctypes.cast(self._row, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i].type not in CONVERT_FUNC: + raise DatabaseError("Invalid data type returned from database") + if data is None: + blocks[i] = None + else: + blocks[i] = CONVERT_FUNC[fields[i].type](data, 1, field_lens[i], precision)[0] + return tuple(blocks) + + +class TaosBlocks: + """TDengine result blocks iterator""" + + def __init__(self, result): + self._result = result + + def __iter__(self): + return self + + def __next__(self): + return self._result.fetch_block() + + def next(self): + return self._result.fetch_block() diff --git a/src/connector/python/taos/statement.py b/src/connector/python/taos/statement.py new file mode 100644 index 0000000000000000000000000000000000000000..155e98173b7f920640aa84d0fcda618d2669bb1e --- /dev/null +++ b/src/connector/python/taos/statement.py @@ -0,0 +1,85 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStmt(object): + """TDengine STMT interface""" + + def __init__(self, stmt, conn = None): + self._conn = conn + self._stmt = stmt + + def set_tbname(self, name): + """Set table name if needed. + + Note that the set_tbname* method should only used in insert statement + """ + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname(self._stmt, name) + + def prepare(self, sql): + # type: (str) -> None + taos_stmt_prepare(self._stmt, sql) + + def set_tbname_tags(self, name, tags): + # type: (str, Array[TaosBind]) -> None + """Set table name with tags, tags is array of BindParams""" + if self._stmt is None: + raise StatementError("Invalid use of set_tbname") + taos_stmt_set_tbname_tags(self._stmt, name, tags) + + def bind_param(self, params, add_batch=True): + # type: (Array[TaosBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param(self._stmt, params) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def bind_param_batch(self, binds, add_batch=True): + # type: (Array[TaosMultiBind], bool) -> None + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_bind_param_batch(self._stmt, binds) + if add_batch: + taos_stmt_add_batch(self._stmt) + + def add_batch(self): + if self._stmt is None: + raise StatementError("Invalid use of stmt") + taos_stmt_add_batch(self._stmt) + + def execute(self): + if self._stmt is None: + raise StatementError("Invalid use of execute") + taos_stmt_execute(self._stmt) + + def use_result(self): + result = taos_stmt_use_result(self._stmt) + return TaosResult(result) + + def close(self): + """Close stmt.""" + if self._stmt is None: + return + taos_stmt_close(self._stmt) + self._stmt = None + + def __del__(self): + self.close() + + +if __name__ == "__main__": + from taos.connection import TaosConnection + + conn = TaosConnection() + + stmt = conn.statement("select * from log.log limit 10") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + stmt.close() + conn.close() diff --git a/src/connector/python/taos/stream.py b/src/connector/python/taos/stream.py new file mode 100644 index 0000000000000000000000000000000000000000..fe3c8c85e3279511972293882224bf20c30dfa64 --- /dev/null +++ b/src/connector/python/taos/stream.py @@ -0,0 +1,22 @@ +from taos.cinterface import * +from taos.error import * +from taos.result import * + + +class TaosStream(object): + """TDengine Stream interface""" + + def __init__(self, stream): + self._raw = stream + + def as_ptr(self): + return self._raw + + def close(self): + """Close stmt.""" + if self._raw is not None: + taos_close_stream(self._raw) + self._raw = None + + def __del__(self): + self.close() diff --git a/src/connector/python/taos/subscription.py b/src/connector/python/taos/subscription.py index 270d9de09217fc58a389981a3542698dd1c0428a..3c6958b6f8d55791b9753a84a4bbd7653bdae780 100644 --- a/src/connector/python/taos/subscription.py +++ b/src/connector/python/taos/subscription.py @@ -1,49 +1,41 @@ -from .cinterface import CTaosInterface +from taos.result import TaosResult +from .cinterface import * from .error import * -class TDengineSubscription(object): - """TDengine subscription object - """ +class TaosSubscription(object): + """TDengine subscription object""" - def __init__(self, sub): + def __init__(self, sub, with_callback = False): self._sub = sub + self._with_callback = with_callback def consume(self): - """Consume rows of a subscription - """ + """Consume rows of a subscription""" if self._sub is None: raise OperationalError("Invalid use of consume") - - result, fields = CTaosInterface.consume(self._sub) - buffer = [[] for i in range(len(fields))] - while True: - block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: - break - for i in range(len(fields)): - buffer[i].extend(block[i]) - - self.fields = fields - return list(map(tuple, zip(*buffer))) + if self._with_callback: + raise OperationalError("DONOT use consume method in an subscription with callback") + result = taos_consume(self._sub) + return TaosResult(result) def close(self, keepProgress=True): - """Close the Subscription. - """ + """Close the Subscription.""" if self._sub is None: return False - CTaosInterface.unsubscribe(self._sub, keepProgress) + taos_unsubscribe(self._sub, keepProgress) + self._sub = None return True + + def __del__(self): + self.close() + +if __name__ == "__main__": + from .connection import TaosConnection -if __name__ == '__main__': - from .connection import TDengineConnection - conn = TDengineConnection( - host="127.0.0.1", - user="root", - password="taosdata", - database="test") + conn = TaosConnection(host="127.0.0.1", user="root", password="taosdata", database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) diff --git a/src/connector/python/taos/timestamp.py b/src/connector/python/taos/timestamp.py new file mode 100644 index 0000000000000000000000000000000000000000..ab5679fdf12e2942aa94f76716ff98e6d2a88d69 --- /dev/null +++ b/src/connector/python/taos/timestamp.py @@ -0,0 +1,17 @@ + +class TimestampType(object): + """Choose which type that parsing TDengine timestamp data to + + - DATETIME: use python datetime.datetime, note that it does not support nanosecond precision, + and python taos will use raw c_int64 as a fallback for nanosecond results. + - NUMPY: use numpy.datetime64 type. + - RAW: use raw c_int64. + - TAOS: use taos' TaosTimestamp. + """ + DATETIME = 0, + NUMPY = 1, + RAW = 2, + TAOS = 3, + +class TaosTimestamp: + pass diff --git a/src/connector/python/tests/test-td6231.py b/src/connector/python/tests/test-td6231.py new file mode 100644 index 0000000000000000000000000000000000000000..e55d22c10734eedcbd5be8012eaeb3fb3d51e381 --- /dev/null +++ b/src/connector/python/tests/test-td6231.py @@ -0,0 +1,50 @@ +from taos import * + +conn = connect() + +dbname = "pytest_taos_stmt_multi" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s" % dbname) +conn.select_db(dbname) + +conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, \ + ti tinyint, si smallint, ii int, bi bigint, tu tinyint unsigned, \ + su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", +) + +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +# params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[14].nchar([None, None, None]) +params[15].timestamp([None, None, 1626861392591]) +stmt.bind_param_batch(params) +stmt.execute() + + +result = stmt.use_result() +assert result.affected_rows == 3 +result.close() + +result = conn.query("select * from log") +for row in result: + print(row) +result.close() +stmt.close() +conn.close() diff --git a/src/connector/python/tests/test_ctaos.py b/src/connector/python/tests/test_ctaos.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9566931f2b29dcbdc8646d2f087ebf40e716cc --- /dev/null +++ b/src/connector/python/tests/test_ctaos.py @@ -0,0 +1,162 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * + +import time +import datetime +import pytest + +@pytest.fixture +def conn(): + return CTaosInterface().connect() + + +def test_simple(conn, caplog): + dbname = "pytest_ctaos_simple" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, level tinyint, content binary(100), ipaddr binary(134))", + ) + taos_free_result(res) + + res = taos_query(conn, "insert into log values(now, 1, 'hello', 'test')") + taos_free_result(res) + + res = taos_query(conn, "select level,content,ipaddr from log limit 1") + + fields = taos_fetch_fields_raw(res) + field_count = taos_field_count(res) + + fields = taos_fetch_fields(res) + for field in fields: + print(field) + + # field_lengths = taos_fetch_lengths(res, field_count) + # if not field_lengths: + # raise "fetch lengths error" + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, field_count) + assert rowstr == "1 hello test" + + row, num = taos_fetch_row(res, fields) + print(row) + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + + +def test_stmt(conn, caplog): + dbname = "pytest_ctaos_stmt" + try: + res = taos_query(conn, "drop database if exists %s" % dbname) + taos_free_result(res) + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100))", + ) + taos_free_result(res) + + stmt = taos_stmt_init(conn) + + taos_stmt_prepare(stmt, "insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + + params = new_bind_params(14) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].null() + params[2].tinyint(2) + params[3].smallint(3) + params[4].int(4) + params[5].bigint(5) + params[6].tinyint_unsigned(6) + params[7].smallint_unsigned(7) + params[8].int_unsigned(8) + params[9].bigint_unsigned(9) + params[10].float(10.1) + params[11].double(10.11) + params[12].binary("hello") + params[13].nchar("stmt") + taos_stmt_bind_param(stmt, params) + taos_stmt_add_batch(stmt) + taos_stmt_execute(stmt) + + res = taos_query(conn, "select * from log limit 1") + + fields = taos_fetch_fields(res) + filed_count = taos_field_count(res) + + row = taos_fetch_row_raw(res) + rowstr = taos_print_row(row, fields, filed_count, 100) + + taos_free_result(res) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + + assert rowstr == "1626861392589 NULL 2 3 4 5 6 7 8 9 10.100000 10.110000 hello stmt" + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err + +def stream_callback(param, result, row): + # type: (c_void_p, c_void_p, c_void_p) -> None + try: + if result == None or row == None: + return + result = c_void_p(result) + row = c_void_p(row) + fields = taos_fetch_fields_raw(result) + num_fields = taos_field_count(result) + s = taos_print_row(row, fields, num_fields) + print(s) + taos_stop_query(result) + except Exception as err: + print(err) + +def test_stream(conn, caplog): + dbname = "pytest_ctaos_stream" + try: + res = taos_query(conn, "create database if not exists %s" % dbname) + taos_free_result(res) + + taos_select_db(conn, dbname) + + res = taos_query( + conn, + "create table if not exists log(ts timestamp, n int)", + ) + taos_free_result(res) + + res = taos_query(conn, "select count(*) from log interval(5s)") + cc = taos_num_fields(res) + assert cc == 2 + + stream = taos_open_stream(conn, "select count(*) from log interval(5s)", stream_callback, 0, None, None) + print("waiting for data") + time.sleep(1) + + for i in range(0, 2): + res = taos_query(conn, "insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + taos_free_result(res) + time.sleep(2) + taos_close_stream(stream) + taos_query(conn, "drop database if exists " + dbname) + taos_close(conn) + except Exception as err: + taos_query(conn, "drop database if exists " + dbname) + raise err diff --git a/src/connector/python/tests/test_info.py b/src/connector/python/tests/test_info.py new file mode 100644 index 0000000000000000000000000000000000000000..bddfec7ef9ddbc203adfcadd262839048466592c --- /dev/null +++ b/src/connector/python/tests/test_info.py @@ -0,0 +1,23 @@ +from taos.cinterface import * + +from taos import * + +import pytest + +@pytest.fixture +def conn(): + return connect() + +def test_client_info(): + print(taos_get_client_info()) + None + +def test_server_info(conn): + # type: (TaosConnection) -> None + print(conn.client_info) + print(conn.server_info) + None + +if __name__ == "__main__": + test_client_info() + test_server_info(connect()) diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9d2cdb39d6f4f2612581ce7284c057c456ef91 --- /dev/null +++ b/src/connector/python/tests/test_lines.py @@ -0,0 +1,57 @@ +from taos.error import OperationalError +from taos import connect, new_bind_params, PrecisionEnum +from taos import * + +from ctypes import * +import taos +import pytest + + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + + +def test_insert_lines(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_insert_lines" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns', + 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + + lines = [ + 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000ns', + ] + conn.insert_lines(lines) + print("inserted") + result = conn.query("select * from st") + print(*result.fields) + all = result.rows_iter() + for row in all: + print(row) + result.close() + print(result.row_count) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err + + +if __name__ == "__main__": + test_insert_lines(connect()) diff --git a/src/connector/python/tests/test_query.py b/src/connector/python/tests/test_query.py new file mode 100644 index 0000000000000000000000000000000000000000..f4e139b1f14df29e8b6304dd2ca03519ea274f43 --- /dev/null +++ b/src/connector/python/tests/test_query.py @@ -0,0 +1,43 @@ +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + return taos.connect() + +def test_query(conn): + """This test will use fetch_block for rows fetching, significantly faster than rows_iter""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result: + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +def test_query_row_iter(conn): + """This test will use fetch_row for each row fetching, this is the only way in async callback""" + result = conn.query("select * from log.log limit 10000") + fields = result.fields + for field in fields: + print("field: %s" % field) + start = datetime.now() + for row in result.rows_iter(): + # print(row) + None + end = datetime.now() + elapsed = end - start + print("elapsed time: ", elapsed) + result.close() + conn.close() + +if __name__ == "__main__": + test_query(taos.connect(database = "log")) + test_query_row_iter(taos.connect(database = "log")) diff --git a/src/connector/python/tests/test_query_a.py b/src/connector/python/tests/test_query_a.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4be5695a87f1fd1017435b13983df7c4f70f06 --- /dev/null +++ b/src/connector/python/tests/test_query_a.py @@ -0,0 +1,66 @@ +from taos import * +from ctypes import * +import taos +import pytest +import time + + +@pytest.fixture +def conn(): + return taos.connect() + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + # print(row) + None + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + + +def query_callback(p_param, p_result, code): + # type: (c_void_p, c_void_p, c_int) -> None + if p_result == None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + # type: (TaosConnection) -> None + counter = Counter(count=0) + conn.query_a("select * from log.log", query_callback, byref(counter)) + + while not counter.done: + print("wait query callback") + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(taos.connect()) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py new file mode 100644 index 0000000000000000000000000000000000000000..938ba10eb3d2377a63f7972deb99dbd47f7de1b2 --- /dev/null +++ b/src/connector/python/tests/test_stmt.py @@ -0,0 +1,149 @@ +from taos import * + +from ctypes import * +from datetime import datetime +import taos +import pytest + +@pytest.fixture +def conn(): + # type: () -> taos.TaosConnection + return connect() + +def test_stmt_insert(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + + stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + params = new_bind_params(16) + params[0].timestamp(1626861392589, PrecisionEnum.Milliseconds) + params[1].bool(True) + params[2].null() + params[3].tinyint(2) + params[4].smallint(3) + params[5].int(4) + params[6].bigint(5) + params[7].tinyint_unsigned(6) + params[8].smallint_unsigned(7) + params[9].int_unsigned(8) + params[10].bigint_unsigned(9) + params[11].float(10.1) + params[12].double(10.11) + params[13].binary("hello") + params[14].nchar("stmt") + params[15].timestamp(1626861392589, PrecisionEnum.Milliseconds) + + stmt.bind_param(params) + stmt.execute() + + result = stmt.use_result() + assert result.affected_rows == 1 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + row = result.next() + print(row) + assert row[2] == None + for i in range(3, 11): + assert row[i] == i - 1 + #float == may not work as expected + # assert row[10] == c_float(10.1) + assert row[12] == 10.11 + assert row[13] == "hello" + assert row[14] == "stmt" + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + +def test_stmt_insert_multi(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_stmt_multi" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + + conn.execute( + "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ + ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", + ) + conn.load_table_info("log") + + start = datetime.now() + stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + + params = new_multi_binds(16) + params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) + params[1].bool((True, None, False)) + params[2].tinyint([-128, -128, None]) # -128 is tinyint null + params[3].tinyint([0, 127, None]) + params[4].smallint([3, None, 2]) + params[5].int([3, 4, None]) + params[6].bigint([3, 4, None]) + params[7].tinyint_unsigned([3, 4, None]) + params[8].smallint_unsigned([3, 4, None]) + params[9].int_unsigned([3, 4, None]) + params[10].bigint_unsigned([3, 4, None]) + params[11].float([3, None, 1]) + params[12].double([3, None, 1.2]) + params[13].binary(["abc", "dddafadfadfadfadfa", None]) + params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) + params[15].timestamp([None, None, 1626861392591]) + stmt.bind_param_batch(params) + + stmt.execute() + end = datetime.now() + print("elapsed time: ", end - start) + result = stmt.use_result() + assert result.affected_rows == 3 + result.close() + stmt.close() + + stmt = conn.statement("select * from log") + stmt.execute() + result = stmt.use_result() + for row in result: + print(row) + result.close() + + stmt.close() + + # start = datetime.now() + # conn.query("insert into log values(1626861392660, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392661, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)(1626861392662, true, NULL, 0, 3,3,3,3,3,3,3,3.0,3.0, 'abc','涛思数据',NULL)") + + # end = datetime.now() + # print("elapsed time: ", end - start) + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err +if __name__ == "__main__": + test_stmt_insert(connect()) + test_stmt_insert_multi(connect()) \ No newline at end of file diff --git a/src/connector/python/tests/test_stream.py b/src/connector/python/tests/test_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..de6e20928b176e51bc6d350fb01268459f4e7f95 --- /dev/null +++ b/src/connector/python/tests/test_stream.py @@ -0,0 +1,70 @@ +from taos.cinterface import * +from taos.precision import * +from taos.bind import * +from taos import * +from ctypes import * +import time +import pytest + + +@pytest.fixture +def conn(): + return connect() + + +def stream_callback(p_param, p_result, p_row): + # type: (c_void_p, c_void_p, c_void_p) -> None + + if p_result == None or p_row == None: + return + result = TaosResult(p_result) + row = TaosRow(result, p_row) + try: + ts, count = row() + p = cast(p_param, POINTER(Counter)) + p.contents.count += count + print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) + + except Exception as err: + print(err) + raise err + + +class Counter(ctypes.Structure): + _fields_ = [ + ("count", c_int), + ] + + def __str__(self): + return "%d" % self.count + + +def test_stream(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_stream" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + result = conn.query("select count(*) from log interval(5s)") + assert result.field_count == 2 + counter = Counter() + counter.count = 0 + stream = conn.stream("select count(*) from log interval(5s)", stream_callback, param=byref(counter)) + + for _ in range(0, 20): + conn.execute("insert into log values(now,0)(now+1s, 1)(now + 2s, 2)") + time.sleep(2) + stream.close() + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_stream(connect()) diff --git a/src/connector/python/tests/test_subscribe.py b/src/connector/python/tests/test_subscribe.py new file mode 100644 index 0000000000000000000000000000000000000000..99fe5b263625c63200f416ec98fcb561773becd8 --- /dev/null +++ b/src/connector/python/tests/test_subscribe.py @@ -0,0 +1,100 @@ +from taos.subscription import TaosSubscription +from taos import * +from ctypes import * +import taos +import pytest +import time +from random import random + + +@pytest.fixture +def conn(): + return taos.connect() + + +def test_subscribe(conn): + # type: (TaosConnection) -> None + + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + + sub = conn.subscribe(True, "test", "select * from log", 1000) + print("# consume from begin") + for ts, n in sub.consume(): + print(ts, n) + + print("# consume new data") + for i in range(5): + conn.execute("insert into log values(now, %d)(now+1s, %d)" % (i, i)) + result = sub.consume() + for ts, n in result: + print(ts, n) + + print("# consume with a stop condition") + for i in range(10): + conn.execute("insert into log values(now, %d)" % int(random() * 10)) + result = sub.consume() + try: + ts, n = next(result) + print(ts, n) + if n > 5: + result.stop_query() + print("## stopped") + break + except StopIteration: + continue + + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +def subscribe_callback(p_sub, p_result, p_param, errno): + # type: (c_void_p, c_void_p, c_void_p, c_int) -> None + print("callback") + result = TaosResult(p_result) + result.check_error(errno) + for row in result.rows_iter(): + ts, n = row() + print(ts, n) + + +def test_subscribe_callback(conn): + # type: (TaosConnection) -> None + dbname = "pytest_taos_subscribe_callback" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s" % dbname) + conn.select_db(dbname) + conn.execute("create table if not exists log(ts timestamp, n int)") + + print("# subscribe with callback") + sub = conn.subscribe(False, "test", "select * from log", 1000, subscribe_callback) + + for i in range(10): + conn.execute("insert into log values(now, %d)" % i) + time.sleep(0.7) + sub.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + raise err + + +if __name__ == "__main__": + test_subscribe(taos.connect()) + test_subscribe_callback(taos.connect()) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4c2ee75c4b9032f24f7271c60255bf40a9a408d6..d64464f948eecd9c24aa65a2161f49feec461488 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -806,7 +806,7 @@ static void printHelp() { "Give this help list"); printf("%s%s%s%s\n", indent, " --usage\t", "\t\t", "Give a short usage message"); - printf("%s%s\n", indent, "-V, --version\t\t\tPrint version info."); + printf("%s%s\n", indent, "-V, --version\t\t\tPrint program version."); /* printf("%s%s%s%s\n", indent, "-D", indent, "Delete database if exists. 0: no, 1: yes, default is 1"); */ @@ -831,6 +831,12 @@ static bool isStringNumber(char *input) return true; } +static void errorUnreconized(char *program, char *wrong_arg) +{ + fprintf(stderr, "%s: unrecognized options '%s'\n", program, wrong_arg); + fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + static void errorPrintReqArg(char *program, char *wrong_arg) { fprintf(stderr, @@ -861,23 +867,31 @@ static void errorPrintReqArg3(char *program, char *wrong_arg) static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "-f") == 0) + if ((0 == strncmp(argv[i], "-f", strlen("-f"))) || (0 == strncmp(argv[i], "--file", strlen("--file")))) { - if (2 == strlen(argv[i])) { - arguments->demo_mode = false; + arguments->demo_mode = false; - if (NULL == argv[i+1]) { - errorPrintReqArg3(argv[0], "f"); + if (2 == strlen(argv[i])) { + if (i+1 == argc) { + errorPrintReqArg(argv[0], "f"); exit(EXIT_FAILURE); } arguments->metaFile = argv[++i]; + } else if (0 == strncmp(argv[i], "-f", strlen("-f"))) { + arguments->metaFile = (char *)(argv[i] + strlen("-f")); } else if (strlen("--file") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--file"); - exit(EXIT_FAILURE); + if (i+1 == argc) { + errorPrintReqArg3(argv[0], "--file"); + exit(EXIT_FAILURE); + } + arguments->metaFile = argv[++i]; } else if (0 == strncmp(argv[i], "--file=", strlen("--file="))) { arguments->metaFile = (char *)(argv[i] + strlen("--file=")); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-c") == 0) + } else if ((0 == strncmp(argv[i], "-c", strlen("-c"))) || (0 == strncmp(argv[i], "--config-dir", strlen("--config-dir")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -885,13 +899,21 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); + } else if (0 == strncmp(argv[i], "-c", strlen("-c"))) { + tstrncpy(configDir, (char *)(argv[i] + strlen("-")), TSDB_FILENAME_LEN); } else if (strlen("--config-dir") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--config-dir"); - exit(EXIT_FAILURE); + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--config-dir"); + exit(EXIT_FAILURE); + } + tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); } else if (0 == strncmp(argv[i], "--config-dir=", strlen("--config-dir="))) { tstrncpy(configDir, (char *)(argv[i] + strlen("--config-dir=")), TSDB_FILENAME_LEN); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-h") == 0) + } else if ((0 == strncmp(argv[i], "-h", strlen("-h"))) || (0 == strncmp(argv[i], "--host", strlen("--host")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -899,13 +921,23 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->host = argv[++i]; + } else if (0 == strncmp(argv[i], "-h", strlen("-h"))) { + arguments->host = (char *)(argv[i] + strlen("-h")); } else if (strlen("--host") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--host"); - exit(EXIT_FAILURE); + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--host"); + exit(EXIT_FAILURE); + } + arguments->host = argv[++i]; } else if (0 == strncmp(argv[i], "--host=", strlen("--host="))) { arguments->host = (char *)(argv[i] + strlen("--host=")); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-P") == 0) + } else if (strcmp(argv[i], "-PP") == 0) { + arguments->performance_print = true; + } else if ((0 == strncmp(argv[i], "-P", strlen("-P"))) || (0 == strncmp(argv[i], "--port", strlen("--port")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -916,15 +948,28 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->port = atoi(argv[++i]); - } else if (strlen("--port") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--port"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--port=", strlen("--port="))) { if (isStringNumber((char *)(argv[i] + strlen("--port=")))) { arguments->port = atoi((char *)(argv[i]+strlen("--port="))); } + } else if (0 == strncmp(argv[i], "-P", strlen("-P"))) { + if (isStringNumber((char *)(argv[i] + strlen("-P")))) { + arguments->port = atoi((char *)(argv[i]+strlen("-P"))); + } + } else if (strlen("--port") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--port"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--port"); + exit(EXIT_FAILURE); + } + arguments->port = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-I") == 0) + } else if ((0 == strncmp(argv[i], "-I", strlen("-I"))) || (0 == strncmp(argv[i], "--interface", strlen("--interface")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -942,9 +987,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } i++; - } else if (strlen("--interface") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--interface"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--interface=", strlen("--interface="))) { if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "taosc")) { arguments->iface = TAOSC_IFACE; @@ -956,8 +998,38 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg3(argv[0], "--interface"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-I", strlen("-I"))) { + if (0 == strcasecmp((char *)(argv[i] + strlen("-I")), "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("-I")), "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("-I")), "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg3(argv[0], "-I"); + exit(EXIT_FAILURE); + } + } else if (strlen("--interface") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--interface"); + exit(EXIT_FAILURE); + } + if (0 == strcasecmp(argv[i+1], "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp(argv[i+1], "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp(argv[i+1], "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg3(argv[0], "--interface"); + exit(EXIT_FAILURE); + } + i++; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-u") == 0) + } else if ((0 == strncmp(argv[i], "-u", strlen("-u"))) || (0 == strncmp(argv[i], "--user", strlen("--user")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -965,13 +1037,21 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->user = argv[++i]; - } else if (strlen("--user") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--user"); - exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "-u", strlen("-u"))) { + arguments->user = (char *)(argv[i++] + strlen("-u")); } else if (0 == strncmp(argv[i], "--user=", strlen("--user="))) { arguments->user = (char *)(argv[i++] + strlen("--user=")); + } else if (strlen("--user") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--user"); + exit(EXIT_FAILURE); + } + arguments->user = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strncmp(argv[i], "-p", 2) == 0) + } else if ((0 == strncmp(argv[i], "-p", strlen("-p"))) || (0 == strcmp(argv[i], "--password"))) { if ((strlen(argv[i]) == 2) || (0 == strcmp(argv[i], "--password"))) { printf("Enter password: "); @@ -983,21 +1063,29 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else { tstrncpy(arguments->password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } - } else if ((strcmp(argv[i], "-o") == 0) + } else if ((0 == strncmp(argv[i], "-o", strlen("-o"))) || (0 == strncmp(argv[i], "--output", strlen("--output")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { - errorPrintReqArg3(argv[0], "o"); + errorPrintReqArg3(argv[0], "--output"); exit(EXIT_FAILURE); } arguments->output_file = argv[++i]; - } else if (strlen("--output") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--output"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--output=", strlen("--output="))) { arguments->output_file = (char *)(argv[i++] + strlen("--output=")); + } else if (0 == strncmp(argv[i], "-o", strlen("-o"))) { + arguments->output_file = (char *)(argv[i++] + strlen("-o")); + } else if (strlen("--output") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--output"); + exit(EXIT_FAILURE); + } + arguments->output_file = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-s") == 0) + } else if ((0 == strncmp(argv[i], "-s", strlen("-s"))) || (0 == strncmp(argv[i], "--sql-file", strlen("--sql-file")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1005,13 +1093,21 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->sqlFile = argv[++i]; - } else if (strlen("--sql-file") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--sql-file"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--sql-file=", strlen("--sql-file="))) { arguments->host = (char *)(argv[i++] + strlen("--sql-file=")); + } else if (0 == strncmp(argv[i], "-s", strlen("-s"))) { + arguments->host = (char *)(argv[i++] + strlen("-s")); + } else if (strlen("--sql-file") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--sql-file"); + exit(EXIT_FAILURE); + } + arguments->sqlFile = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-q") == 0) + } else if ((0 == strncmp(argv[i], "-q", strlen("-q"))) || (0 == strncmp(argv[i], "--query-mode", strlen("--query-mode")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1022,9 +1118,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->async_mode = atoi(argv[++i]); - } else if (strlen("--query-mode") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--query-mode"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--query-mode=", strlen("--query-mode="))) { if (isStringNumber((char *)(argv[i] + strlen("--query-mode=")))) { arguments->async_mode = atoi((char *)(argv[i]+strlen("--query-mode="))); @@ -1032,8 +1125,27 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--query-mode"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-q", strlen("-q"))) { + if (isStringNumber((char *)(argv[i] + strlen("-q")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("-q"))); + } else { + errorPrintReqArg2(argv[0], "-q"); + exit(EXIT_FAILURE); + } + } else if (strlen("--query-mode") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--query-mode"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--query-mode"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-T") == 0) + } else if ((0 == strncmp(argv[i], "-T", strlen("-T"))) || (0 == strncmp(argv[i], "--threads", strlen("--threads")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1044,9 +1156,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->num_of_threads = atoi(argv[++i]); - } else if (strlen("--threads") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--threads"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--threads=", strlen("--threads="))) { if (isStringNumber((char *)(argv[i] + strlen("--threads=")))) { arguments->num_of_threads = atoi((char *)(argv[i]+strlen("--threads="))); @@ -1054,8 +1163,27 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--threads"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-T", strlen("-T"))) { + if (isStringNumber((char *)(argv[i] + strlen("-T")))) { + arguments->num_of_threads = atoi((char *)(argv[i]+strlen("-T"))); + } else { + errorPrintReqArg2(argv[0], "-T"); + exit(EXIT_FAILURE); + } + } else if (strlen("--threads") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--threads"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--threads"); + exit(EXIT_FAILURE); + } + arguments->num_of_threads = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-i") == 0) + } else if ((0 == strncmp(argv[i], "-i", strlen("-i"))) || (0 == strncmp(argv[i], "--insert-interval", strlen("--insert-interval")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1066,25 +1194,34 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); - } else if (strlen("--insert-interval")== strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--insert-interval"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--insert-interval=", strlen("--insert-interval="))) { - if (isStringNumber((char *)(argv[i] + 18))) { + if (isStringNumber((char *)(argv[i] + strlen("--insert-interval=")))) { arguments->insert_interval = atoi((char *)(argv[i]+strlen("--insert-interval="))); } else { errorPrintReqArg3(argv[0], "--insert-innterval"); exit(EXIT_FAILURE); } - } - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-i need a number following!\n"); + } else if (0 == strncmp(argv[i], "-i", strlen("-i"))) { + if (isStringNumber((char *)(argv[i] + strlen("-i")))) { + arguments->insert_interval = atoi((char *)(argv[i]+strlen("-i"))); + } else { + errorPrintReqArg3(argv[0], "-i"); + exit(EXIT_FAILURE); + } + } else if (strlen("--insert-interval")== strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--insert-interval"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--insert-interval"); + exit(EXIT_FAILURE); + } + arguments->insert_interval = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); exit(EXIT_FAILURE); } - arguments->insert_interval = atoi(argv[++i]); - } else if ((strcmp(argv[i], "-S") == 0) + } else if ((0 == strncmp(argv[i], "-S", strlen("-S"))) || (0 == strncmp(argv[i], "--time-step", strlen("--time-step")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1095,9 +1232,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->async_mode = atoi(argv[++i]); - } else if (strlen("--time-step") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--time-step"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--time-step=", strlen("--time-step="))) { if (isStringNumber((char *)(argv[i] + strlen("--time-step=")))) { arguments->async_mode = atoi((char *)(argv[i]+strlen("--time-step="))); @@ -1105,6 +1239,25 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--time-step"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-S", strlen("-S"))) { + if (isStringNumber((char *)(argv[i] + strlen("-S")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("-S"))); + } else { + errorPrintReqArg2(argv[0], "-S"); + exit(EXIT_FAILURE); + } + } else if (strlen("--time-step") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--time-step"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--time-step"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } } else if (strcmp(argv[i], "-qt") == 0) { if ((argc == i+1) @@ -1114,9 +1267,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); - } else if ((strcmp(argv[i], "-B") == 0) + } else if ((0 == strncmp(argv[i], "-B", strlen("-B"))) || (0 == strncmp(argv[i], "--interlace-rows", strlen("--interlace-rows")))) { - if (2 == strlen(argv[i])) { + if (strlen("-B") == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg(argv[0], "B"); exit(EXIT_FAILURE); @@ -1125,9 +1278,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->interlace_rows = atoi(argv[++i]); - } else if (strlen("--interlace-rows")== strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--interlace-rows"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--interlace-rows=", strlen("--interlace-rows="))) { if (isStringNumber((char *)(argv[i] + strlen("--interlace-rows=")))) { arguments->interlace_rows = atoi((char *)(argv[i]+strlen("--interlace-rows="))); @@ -1135,10 +1285,29 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--interlace-rows"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-B", strlen("-B"))) { + if (isStringNumber((char *)(argv[i] + strlen("-B")))) { + arguments->interlace_rows = atoi((char *)(argv[i]+strlen("-B"))); + } else { + errorPrintReqArg2(argv[0], "-B"); + exit(EXIT_FAILURE); + } + } else if (strlen("--interlace-rows")== strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--interlace-rows"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--interlace-rows"); + exit(EXIT_FAILURE); + } + arguments->interlace_rows = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-r") == 0) + } else if ((0 == strncmp(argv[i], "-r", strlen("-r"))) || (0 == strncmp(argv[i], "--rec-per-req", 13))) { - if (2 == strlen(argv[i])) { + if (strlen("-r") == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg(argv[0], "r"); exit(EXIT_FAILURE); @@ -1147,9 +1316,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->num_of_RPR = atoi(argv[++i]); - } else if (strlen("--rec-per-req")== strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--rec-per-req"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--rec-per-req=", strlen("--rec-per-req="))) { if (isStringNumber((char *)(argv[i] + strlen("--rec-per-req=")))) { arguments->num_of_RPR = atoi((char *)(argv[i]+strlen("--rec-per-req="))); @@ -1157,8 +1323,27 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--rec-per-req"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-r", strlen("-r"))) { + if (isStringNumber((char *)(argv[i] + strlen("-r")))) { + arguments->num_of_RPR = atoi((char *)(argv[i]+strlen("-r"))); + } else { + errorPrintReqArg2(argv[0], "-r"); + exit(EXIT_FAILURE); + } + } else if (strlen("--rec-per-req")== strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--rec-per-req"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--rec-per-req"); + exit(EXIT_FAILURE); + } + arguments->num_of_RPR = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-t") == 0) + } else if ((0 == strncmp(argv[i], "-t", strlen("-t"))) || (0 == strncmp(argv[i], "--tables", strlen("--tables")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1169,9 +1354,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->num_of_tables = atoi(argv[++i]); - } else if (strlen("--tables") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--tables"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--tables=", strlen("--tables="))) { if (isStringNumber((char *)(argv[i] + strlen("--tables=")))) { arguments->num_of_tables = atoi((char *)(argv[i]+strlen("--tables="))); @@ -1179,8 +1361,29 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--tables"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-t", strlen("-t"))) { + if (isStringNumber((char *)(argv[i] + strlen("-t")))) { + arguments->num_of_tables = atoi((char *)(argv[i]+strlen("-t"))); + } else { + errorPrintReqArg2(argv[0], "-t"); + exit(EXIT_FAILURE); + } + } else if (strlen("--tables") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--tables"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--tables"); + exit(EXIT_FAILURE); + } + arguments->num_of_tables = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-n") == 0) + + g_totalChildTables = arguments->num_of_tables; + } else if ((0 == strncmp(argv[i], "-n", strlen("-n"))) || (0 == strncmp(argv[i], "--records", strlen("--records")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1191,9 +1394,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->num_of_DPT = atoi(argv[++i]); - } else if (strlen("--records") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--records"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--records=", strlen("--records="))) { if (isStringNumber((char *)(argv[i] + strlen("--records=")))) { arguments->num_of_DPT = atoi((char *)(argv[i]+strlen("--records="))); @@ -1201,25 +1401,49 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--records"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-n", strlen("-n"))) { + if (isStringNumber((char *)(argv[i] + strlen("-n")))) { + arguments->num_of_DPT = atoi((char *)(argv[i]+strlen("-n"))); + } else { + errorPrintReqArg2(argv[0], "-n"); + exit(EXIT_FAILURE); + } + } else if (strlen("--records") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--records"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--records"); + exit(EXIT_FAILURE); + } + arguments->num_of_DPT = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - - g_totalChildTables = arguments->num_of_DPT; - } else if ((strcmp(argv[i], "-d") == 0) - + } else if ((0 == strncmp(argv[i], "-d", strlen("-d"))) || (0 == strncmp(argv[i], "--database", strlen("--database")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { - errorPrintReqArg3(argv[0], "d"); + errorPrintReqArg(argv[0], "d"); exit(EXIT_FAILURE); } arguments->database = argv[++i]; - } else if (strlen("--database") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--database"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--database=", strlen("--database="))) { arguments->output_file = (char *)(argv[i] + strlen("--database=")); + } else if (0 == strncmp(argv[i], "-d", strlen("-d"))) { + arguments->output_file = (char *)(argv[i] + strlen("-d")); + } else if (strlen("--database") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--database"); + exit(EXIT_FAILURE); + } + arguments->database = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-l") == 0) + } else if ((0 == strncmp(argv[i], "-l", strlen("-l"))) || (0 == strncmp(argv[i], "--columns", strlen("--columns")))) { arguments->demo_mode = false; if (2 == strlen(argv[i])) { @@ -1231,16 +1455,32 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->num_of_CPR = atoi(argv[++i]); - } else if (strlen("--columns")== strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--columns"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--columns=", strlen("--columns="))) { if (isStringNumber((char *)(argv[i] + strlen("--columns=")))) { - arguments->num_of_CPR = atoi((char *)(argv[i]+strlen("--columns"))); + arguments->num_of_CPR = atoi((char *)(argv[i]+strlen("--columns="))); } else { errorPrintReqArg2(argv[0], "--columns"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-l", strlen("-l"))) { + if (isStringNumber((char *)(argv[i] + strlen("-l")))) { + arguments->num_of_CPR = atoi((char *)(argv[i]+strlen("-l"))); + } else { + errorPrintReqArg2(argv[0], "-l"); + exit(EXIT_FAILURE); + } + } else if (strlen("--columns")== strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--columns"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--columns"); + exit(EXIT_FAILURE); + } + arguments->num_of_CPR = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } if (arguments->num_of_CPR > MAX_NUM_COLUMNS) { @@ -1255,7 +1495,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } - } else if ((strcmp(argv[i], "-b") == 0) + } else if ((0 == strncmp(argv[i], "-b", strlen("-b"))) || (0 == strncmp(argv[i], "--data-type", strlen("--data-type")))) { arguments->demo_mode = false; @@ -1266,11 +1506,19 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } dataType = argv[++i]; - } else if (strlen("--data-type") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--data-type"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--data-type=", strlen("--data-type="))) { dataType = (char *)(argv[i] + strlen("--data-type=")); + } else if (0 == strncmp(argv[i], "-b", strlen("-b"))) { + dataType = (char *)(argv[i] + strlen("-b")); + } else if (strlen("--data-type") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--data-type"); + exit(EXIT_FAILURE); + } + dataType = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } if (strstr(dataType, ",") == NULL) { @@ -1319,7 +1567,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->datatype[index] = NULL; } - } else if ((strcmp(argv[i], "-w") == 0) + } else if ((0 == strncmp(argv[i], "-w", strlen("-w"))) || (0 == strncmp(argv[i], "--binwidth", strlen("--binwidth")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1330,9 +1578,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->binwidth = atoi(argv[++i]); - } else if (strlen("--binwidth") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--binwidth"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--binwidth=", strlen("--binwidth="))) { if (isStringNumber((char *)(argv[i] + strlen("--binwidth=")))) { arguments->binwidth = atoi((char *)(argv[i]+strlen("--binwidth="))); @@ -1340,21 +1585,47 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--binwidth"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-w", strlen("-w"))) { + if (isStringNumber((char *)(argv[i] + strlen("-w")))) { + arguments->binwidth = atoi((char *)(argv[i]+strlen("-w"))); + } else { + errorPrintReqArg2(argv[0], "-w"); + exit(EXIT_FAILURE); + } + } else if (strlen("--binwidth") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--binwidth"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--binwidth"); + exit(EXIT_FAILURE); + } + arguments->binwidth = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-m") == 0) + } else if ((0 == strncmp(argv[i], "-m", strlen("-m"))) || (0 == strncmp(argv[i], "--table-prefix", strlen("--table-prefix")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { - errorPrintReqArg3(argv[0], "m"); + errorPrintReqArg(argv[0], "m"); exit(EXIT_FAILURE); } arguments->tb_prefix = argv[++i]; - } else if (strlen("--table-prefix") == strlen(argv[i]) - || (strlen("--table-prefix=") == strlen(argv[i]))) { - errorPrintReqArg3(argv[0], "--table-prefix"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--table-prefix=", strlen("--table-prefix="))) { arguments->tb_prefix = (char *)(argv[i] + strlen("--table-prefix=")); + } else if (0 == strncmp(argv[i], "-m", strlen("-m"))) { + arguments->tb_prefix = (char *)(argv[i] + strlen("-m")); + } else if (strlen("--table-prefix") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--table-prefix"); + exit(EXIT_FAILURE); + } + arguments->tb_prefix = argv[++i]; + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } } else if ((strcmp(argv[i], "-N") == 0) || (0 == strcmp(argv[i], "--normal-table"))) { @@ -1373,9 +1644,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->debug_print = true; } else if (strcmp(argv[i], "-gg") == 0) { arguments->verbose_print = true; - } else if (strcmp(argv[i], "-PP") == 0) { - arguments->performance_print = true; - } else if ((strcmp(argv[i], "-O") == 0) + } else if ((0 == strncmp(argv[i], "-O", strlen("-O"))) || (0 == strncmp(argv[i], "--disorder", strlen("--disorder")))) { if (2 == strlen(argv[i])) { if (argc == i+1) { @@ -1386,9 +1655,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->disorderRatio = atoi(argv[++i]); - } else if (strlen("--disorder") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--disorder"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--disorder=", strlen("--disorder="))) { if (isStringNumber((char *)(argv[i] + strlen("--disorder=")))) { arguments->disorderRatio = atoi((char *)(argv[i]+strlen("--disorder="))); @@ -1396,6 +1662,25 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--disorder"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-O", strlen("-O"))) { + if (isStringNumber((char *)(argv[i] + strlen("-O")))) { + arguments->disorderRatio = atoi((char *)(argv[i]+strlen("-O"))); + } else { + errorPrintReqArg2(argv[0], "-O"); + exit(EXIT_FAILURE); + } + } else if (strlen("--disorder") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--disorder"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--disorder"); + exit(EXIT_FAILURE); + } + arguments->disorderRatio = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } if (arguments->disorderRatio > 50) { @@ -1409,7 +1694,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->disorderRatio, 0); arguments->disorderRatio = 0; } - } else if ((strcmp(argv[i], "-R") == 0) + } else if ((0 == strncmp(argv[i], "-R", strlen("-R"))) || (0 == strncmp(argv[i], "--disorder-range", strlen("--disorder-range")))) { if (2 == strlen(argv[i])) { @@ -1421,9 +1706,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->disorderRange = atoi(argv[++i]); - } else if (strlen("--disorder-range") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--disorder-range"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--disorder-range=", strlen("--disorder-range="))) { if (isStringNumber((char *)(argv[i] + strlen("--disorder-range=")))) { @@ -1433,14 +1715,34 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--disorder-range"); exit(EXIT_FAILURE); } - } + } else if (0 == strncmp(argv[i], "-R", strlen("-R"))) { + if (isStringNumber((char *)(argv[i] + strlen("-R")))) { + arguments->disorderRange = + atoi((char *)(argv[i]+strlen("-R"))); + } else { + errorPrintReqArg2(argv[0], "-R"); + exit(EXIT_FAILURE); + } - if (arguments->disorderRange < 0) { - errorPrint("Invalid disorder range %d, will be set to %d\n", - arguments->disorderRange, 1000); - arguments->disorderRange = 1000; + if (arguments->disorderRange < 0) { + errorPrint("Invalid disorder range %d, will be set to %d\n", + arguments->disorderRange, 1000); + arguments->disorderRange = 1000; + } + } else if (strlen("--disorder-range") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--disorder-range"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--disorder-range"); + exit(EXIT_FAILURE); + } + arguments->disorderRange = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "-a") == 0) + } else if ((0 == strncmp(argv[i], "-a", strlen("-a"))) || (0 == strncmp(argv[i], "--replica", strlen("--replica")))) { if (2 == strlen(argv[i])) { @@ -1452,9 +1754,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->replica = atoi(argv[++i]); - } else if (strlen("--replica") == strlen(argv[i])) { - errorPrintReqArg3(argv[0], "--replica"); - exit(EXIT_FAILURE); } else if (0 == strncmp(argv[i], "--replica=", strlen("--replica="))) { if (isStringNumber((char *)(argv[i] + strlen("--replica=")))) { @@ -1464,7 +1763,28 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrintReqArg2(argv[0], "--replica"); exit(EXIT_FAILURE); } + } else if (0 == strncmp(argv[i], "-a", strlen("-a"))) { + if (isStringNumber((char *)(argv[i] + strlen("-a")))) { + arguments->replica = + atoi((char *)(argv[i]+strlen("-a"))); + } else { + errorPrintReqArg2(argv[0], "-a"); + exit(EXIT_FAILURE); + } + } else if (strlen("--replica") == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "--replica"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "--replica"); + exit(EXIT_FAILURE); + } + arguments->replica = atoi(argv[++i]); + } else { + errorUnreconized(argv[0], argv[i]); + exit(EXIT_FAILURE); } + if (arguments->replica > 3 || arguments->replica < 1) { errorPrint("Invalid replica value %d, will be set to %d\n", arguments->replica, 1); @@ -1496,7 +1816,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else { // to simulate argp_option output if (strlen(argv[i]) > 2) { - fprintf(stderr, "%s unrecognized options '%s'\n", argv[0], argv[i]); + if (0 == strncmp(argv[i], "--", 2)) { + fprintf(stderr, "%s: unrecognized options '%s'\n", argv[0], argv[i]); + } else if (0 == strncmp(argv[i], "-", 1)) { + char tmp[2] = {0}; + tstrncpy(tmp, argv[i]+1, 2); + fprintf(stderr, "%s: invalid options -- '%s'\n", argv[0], tmp); + } else { + fprintf(stderr, "%s: Too many arguments\n", argv[0]); + } } else { fprintf(stderr, "%s invalid options -- '%s'\n", argv[0], (char *)((char *)argv[i])+1); @@ -1531,9 +1859,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->use_metric ? "true" : "false"); if (*(arguments->datatype)) { printf("# Specified data type: "); - for (int i = 0; i < MAX_NUM_COLUMNS; i++) - if (arguments->datatype[i]) - printf("%s,", arguments->datatype[i]); + for (int c = 0; c < MAX_NUM_COLUMNS; c++) + if (arguments->datatype[c]) + printf("%s,", arguments->datatype[c]); else break; printf("\n"); @@ -5648,12 +5976,85 @@ static int64_t generateData(char *recBuf, char **data_type, return (int32_t)strlen(recBuf); } +static int generateSampleMemoryFromRand(SSuperTable *stbInfo) +{ + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + char *buff = malloc(stbInfo->lenOfOneRow); + if (NULL == buff) { + errorPrint2("%s() LN%d, memory allocation %"PRId64" bytes failed\n", + __func__, __LINE__, stbInfo->lenOfOneRow); + exit(EXIT_FAILURE); + } + + for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) { + uint64_t pos = 0; + memset(buff, 0, stbInfo->lenOfOneRow); + + for (int c = 0; c < stbInfo->columnCount; c++) { + char *tmp; + if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BINARY", strlen("BINARY"))) { + rand_string(data, stbInfo->columns[c].dataLen); + pos += sprintf(buff + pos, "%s,", data); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "NCHAR", strlen("NCHAR"))) { + rand_string(data, stbInfo->columns[c].dataLen); + pos += sprintf(buff + pos, "%s,", data); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "INT", strlen("INT"))) { + if ((g_args.demo_mode) && (c == 1)) { + tmp = demo_voltage_int_str(); + } else { + tmp = rand_int_str(); + } + pos += sprintf(buff + pos, "%s,", tmp); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BIGINT", strlen("BIGINT"))) { + pos += sprintf(buff + pos, "%s,", rand_bigint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "FLOAT", strlen("FLOAT"))) { + if (g_args.demo_mode) { + if (c == 0) { + tmp = demo_current_float_str(); + } else { + tmp = demo_phase_float_str(); + } + } else { + tmp = rand_float_str(); + } + pos += sprintf(buff + pos, "%s,", tmp); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "DOUBLE", strlen("DOUBLE"))) { + pos += sprintf(buff + pos, "%s,", rand_double_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "SMALLINT", strlen("SMALLINT"))) { + pos += sprintf(buff + pos, "%s,", rand_smallint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "TINYINT", strlen("TINYINT"))) { + pos += sprintf(buff + pos, "%s,", rand_tinyint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BOOL", strlen("BOOL"))) { + pos += sprintf(buff + pos, "%s,", rand_bool_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "TIMESTAMP", strlen("TIMESTAMP"))) { + pos += sprintf(buff + pos, "%s,", rand_bigint_str()); + } + } + *(buff + pos - 1) = 0; + memcpy(stbInfo->sampleDataBuf + i * stbInfo->lenOfOneRow, buff, pos); + } + + free(buff); + return 0; +} + static int prepareSampleDataForSTable(SSuperTable *stbInfo) { - char* sampleDataBuf = NULL; - sampleDataBuf = calloc( + stbInfo->sampleDataBuf = calloc( stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); - if (sampleDataBuf == NULL) { + if (NULL == stbInfo->sampleDataBuf) { errorPrint2("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", __func__, __LINE__, stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, @@ -5661,13 +6062,16 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) { return -1; } - stbInfo->sampleDataBuf = sampleDataBuf; - int ret = readSampleFromCsvFileToMem(stbInfo); + int ret; + if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample"))) + ret = readSampleFromCsvFileToMem(stbInfo); + else + ret = generateSampleMemoryFromRand(stbInfo); if (0 != ret) { errorPrint2("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__); - tmfree(sampleDataBuf); + tmfree(stbInfo->sampleDataBuf); stbInfo->sampleDataBuf = NULL; return -1; } @@ -7385,11 +7789,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pstr += len; remainderBufLen -= len; + // measure prepare + insert + startTs = taosGetTimestampUs(); + int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { if (sourceRand) { - generated = prepareStbStmtRand( +/* generated = prepareStbStmtRand( pThreadInfo, tableName, tableSeq, @@ -7397,6 +7804,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { insertRows, i, start_time ); + */ + generated = prepareStbStmtWithSample( + pThreadInfo, + tableName, + tableSeq, + g_args.num_of_RPR, + insertRows, i, start_time, + &(pThreadInfo->samplePos)); } else { generated = prepareStbStmtWithSample( pThreadInfo, @@ -7441,7 +7856,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; - startTs = taosGetTimestampUs(); + // only measure insert + // startTs = taosGetTimestampUs(); int32_t affectedRows = execInsert(pThreadInfo, generated); @@ -7651,7 +8067,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return -1; } - for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) { char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { @@ -7660,7 +8075,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return -1; } - TAOS_BIND *bind; int cursor = 0; @@ -7749,11 +8163,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, debugPrint("%s() LN%d, start_time= %"PRId64"\n", __func__, __LINE__, start_time); - int64_t start = taosGetTimestampMs(); - // read sample data from file first - if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource, - "sample", strlen("sample")))) { + if (stbInfo) { if (0 != prepareSampleDataForSTable(stbInfo)) { errorPrint2("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__); @@ -7901,8 +8312,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, debugPrint("%s() LN%d, stmtBuffer: %s", __func__, __LINE__, stmtBuffer); - if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource, - "sample", strlen("sample")))) { + if (stbInfo) { parseSampleFileToStmt(stbInfo, timePrec); } } @@ -7987,6 +8397,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, free(stmtBuffer); + int64_t start = taosGetTimestampMs(); + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -8029,22 +8441,22 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (cntDelay == 0) cntDelay = 1; avgDelay = (double)totalDelay / cntDelay; - int64_t end = taosGetTimestampMs(); + int64_t end = taosGetTimestampMs(); int64_t t = end - start; - double tInMs = t/1000.0; + double tInMs = (double) t / 1000.0; if (stbInfo) { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", tInMs, stbInfo->totalInsertRows, stbInfo->totalAffectedRows, threads, db_name, stbInfo->sTblName, - (tInMs)? + (double) tInMs? (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", tInMs, stbInfo->totalInsertRows, stbInfo->totalAffectedRows, threads, db_name, stbInfo->sTblName, @@ -8052,7 +8464,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX); } } else { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", tInMs, g_args.totalInsertRows, g_args.totalAffectedRows, threads, db_name, @@ -8060,7 +8472,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, (double)(g_args.totalInsertRows/tInMs):FLT_MAX); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", tInMs, g_args.totalInsertRows, g_args.totalAffectedRows, threads, db_name, diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2bd5723d5a60169fc7417daea2825fdd2a240716..6d60e201bb8467ef59f4b4edf8e734babf553cd8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -231,6 +231,12 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); +static void doSetOperatorCompleted(SOperatorInfo* pOperator) { + pOperator->status = OP_EXEC_DONE; + if (pOperator->pRuntimeEnv != NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + } +} static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); @@ -5313,7 +5319,7 @@ SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { // TSDB_FUNC_TAG_DUMMY function needs to be ignored if (index->colId == pExpr->colInfo.colId && - ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_TAG) || + ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && ((pExpr->functionId == TSDB_FUNC_TAG) || (pExpr->functionId == TSDB_FUNC_TAGPRJ))) || (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && pExpr->functionId == TSDB_FUNC_PRJ))) { index->colIndex = j; index->colId = pExpr->resColId; @@ -5493,8 +5499,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { // start to flush data into disk and try do multiway merge sort if (pBlock == NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); break; } @@ -5605,8 +5610,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); } - pOperator->status = OP_EXEC_DONE; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + doSetOperatorCompleted(pOperator); finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); pInfo->pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); @@ -5682,7 +5686,7 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } return pInfo->pRes; @@ -5800,8 +5804,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); return NULL; } @@ -5829,8 +5832,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); pInfo->total = pInfo->limit; - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } else { pInfo->total += pBlock->info.rows; } @@ -5865,8 +5867,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { } } - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); return NULL; } @@ -5881,9 +5882,8 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } return pIntervalInfo->pRes; @@ -5924,7 +5924,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; @@ -5943,7 +5943,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } return pIntervalInfo->pRes; @@ -6002,7 +6002,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); } return pIntervalInfo->pRes; @@ -7250,13 +7250,11 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); break; } if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); break; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 71a9009c4581dae86861a67de850e275cae352f7..e3ad254bbb1a4ce0a3c504e9a799a109b7e1159a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3474,19 +3474,19 @@ void filterPrepare(void* expr, void* param) { if (pInfo->optr == TSDB_RELATION_IN) { int dummy = -1; - SHashObj *pObj = NULL; + SHashObj *pObj = NULL; if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { pObj = taosHashInit(256, taosGetDefaultHashFunction(pInfo->sch.type), true, false); SArray *arr = (SArray *)(pCond->arr); for (size_t i = 0; i < taosArrayGetSize(arr); i++) { char* p = taosArrayGetP(arr, i); - strtolower(varDataVal(p), varDataVal(p)); - taosHashPut(pObj, varDataVal(p),varDataLen(p), &dummy, sizeof(dummy)); + strntolower_s(varDataVal(p), varDataVal(p), varDataLen(p)); + taosHashPut(pObj, varDataVal(p), varDataLen(p), &dummy, sizeof(dummy)); } } else { buildFilterSetFromBinary((void **)&pObj, pCond->pz, pCond->nLen); } - pInfo->q = (char *)pObj; + pInfo->q = (char *)pObj; } else if (pCond != NULL) { uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE; if (size < (uint32_t)pSchema->bytes) { diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 7c8fd2ed8353a086ec9f145aaaa30f4db57113cb..6bcfb5de295c5719032b81c23d16ec2b1476349e 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -32,6 +32,7 @@ char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); char ** strsplit(char *src, const char *delim, int32_t *num); char * strtolower(char *dst, const char *src); char * strntolower(char *dst, const char *src, int32_t n); +char * strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); char * strbetween(char *string, char *begin, char *end); char * paGetToken(char *src, char **token, int32_t *tokenLen); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 1a73991ade1ea4617fc4d3dab3904652ff46d691..f8a97ff7cd04ac6888806f69f53dbc0188aaa8cd 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -197,7 +197,7 @@ char* strntolower(char *dst, const char *src, int32_t n) { if (n == 0) { *p = 0; return dst; - } + } for (c = *src++; n-- > 0; c = *src++) { if (esc) { esc = 0; @@ -219,6 +219,26 @@ char* strntolower(char *dst, const char *src, int32_t n) { return dst; } +char* strntolower_s(char *dst, const char *src, int32_t n) { + char *p = dst, c; + + assert(dst != NULL); + if (n == 0) { + return NULL; + } + + while (n-- > 0) { + c = *src; + if (c >= 'A' && c <= 'Z') { + c -= 'A' - 'a'; + } + *p++ = c; + src++; + } + + return dst; +} + char *paGetToken(char *string, char **token, int32_t *tokenLen) { char quote = 0; diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7c916c0a3ad1c8140193cae5852ec0456309b989..f2312821b5f7c221ea86e27c96c74d988db92d8e 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -387,6 +387,7 @@ python3 ./test.py -f insert/schemalessInsert.py python3 ./test.py -f alter/alterColMultiTimes.py python3 ./test.py -f query/queryWildcardLength.py python3 ./test.py -f query/queryTbnameUpperLower.py +python3 ./test.py -f query/queryGroupTbname.py #======================p4-end=============== diff --git a/tests/pytest/insert/line_insert.py b/tests/pytest/insert/line_insert.py index 53eaa55aa50a1369b4aff9c49421263788205038..92fdd0f28e612994df414ea1b560152a3f2001a8 100644 --- a/tests/pytest/insert/line_insert.py +++ b/tests/pytest/insert/line_insert.py @@ -42,18 +42,18 @@ class TDTestCase: "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines(lines) + print("insert_lines result {}".format(code)) lines2 = [ "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns" ] - code = self._conn.insertLines([ lines2[0] ]) - print("insertLines result {}".format(code)) + code = self._conn.insert_lines([ lines2[0] ]) + print("insert_lines result {}".format(code)) - self._conn.insertLines([ lines2[1] ]) - print("insertLines result {}".format(code)) + self._conn.insert_lines([ lines2[1] ]) + print("insert_lines result {}".format(code)) tdSql.query("select * from st") tdSql.checkRows(4) @@ -73,7 +73,7 @@ class TDTestCase: tdSql.query("describe stf") tdSql.checkData(2, 2, 14) - self._conn.insertLines([ + self._conn.insert_lines([ "sth,t1=4i64,t2=5f64,t4=5f64,ID=\"childtable\" c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641ms", "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654ms" ]) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index fc95f860bcc2d73826d908dd67c0c6f6e62f31cc..8f1e01acd41b912de46520e109808d4677db6d9d 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -11,8 +11,10 @@ # -*- coding: utf-8 -*- +import traceback import random import string +from taos.error import LinesError import time from copy import deepcopy import numpy as np @@ -292,7 +294,7 @@ class TDTestCase: def resCmp(self, input_sql, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None): expect_list = self.inputHandle(input_sql) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) query_sql = f"{query_sql} {stb_name} {condition}" res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) if ts == 0: @@ -312,7 +314,9 @@ class TDTestCase: expect_list[0].pop(j) tdSql.checkEqual(res_row_list[0], expect_list[0]) tdSql.checkEqual(res_field_list_without_ts, expect_list[1]) - tdSql.checkEqual(res_type_list, expect_list[2]) + for i in range(len(res_type_list)): + tdSql.checkEqual(res_type_list[i], expect_list[2][i]) + # tdSql.checkEqual(res_type_list, expect_list[2]) def cleanStb(self): query_sql = "show stables" @@ -405,13 +409,14 @@ class TDTestCase: """ for input_sql in [self.genLongSql(128, 1)[0], self.genLongSql(1, 4094)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: self.cleanStb() - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) - + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass + def idIllegalNameCheckCase(self): """ test illegal id name @@ -421,8 +426,10 @@ class TDTestCase: rstr = list("`~!@#$¥%^&*()-+={}|[]、「」【】\:;《》<>?") for i in rstr: input_sql = self.genFullTypeSql(tb_name=f"\"aaa{i}bbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def idStartWithNumCheckCase(self): """ @@ -430,8 +437,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(tb_name=f"\"1aaabbb\"")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def nowTsCheckCase(self): """ @@ -439,8 +448,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="now")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def dateFormatTsCheckCase(self): """ @@ -448,8 +459,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="2021-07-21\ 19:01:46.920")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def illegalTsCheckCase(self): """ @@ -457,8 +470,10 @@ class TDTestCase: """ self.cleanStb() input_sql = self.genFullTypeSql(ts="16260068336390us19")[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass def tagValueLengthCheckCase(self): """ @@ -471,8 +486,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(t1=t1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i16 for t2 in ["-32767i16", "32767i16"]: @@ -480,8 +497,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(t2=t2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i32 for t3 in ["-2147483647i32", "2147483647i32"]: @@ -489,8 +508,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(t3=t3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass #i64 for t4 in ["-9223372036854775807i64", "9223372036854775807i64"]: @@ -498,8 +519,10 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for t4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(t4=t4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + except LinesError: + pass # f32 for t5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -508,8 +531,12 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for t5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(t5=t5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) + # f64 for t6 in [f'{-1.79769*(10**308)}f64', f'{-1.79769*(10**308)}f64']: @@ -518,27 +545,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16375, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + pass # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4094, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def colValueLengthCheckCase(self): """ @@ -552,16 +588,22 @@ class TDTestCase: for c1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(c1=c1)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i16 for c2 in ["-32767i16"]: input_sql, stb_name = self.genFullTypeSql(c2=c2) self.resCmp(input_sql, stb_name) for c2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(c2=c2)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i32 for c3 in ["-2147483647i32"]: @@ -569,8 +611,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(c3=c3)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i64 for c4 in ["-9223372036854775807i64"]: @@ -578,8 +623,11 @@ class TDTestCase: self.resCmp(input_sql, stb_name) for c4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(c4=c4)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f32 for c5 in [f"{-3.4028234663852885981170418348451692544*(10**38)}f32", f"{3.4028234663852885981170418348451692544*(10**38)}f32"]: @@ -588,8 +636,11 @@ class TDTestCase: # * limit set to 4028234664*(10**38) for c5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(c5=c5)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # f64 for c6 in [f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64', f'{-1.79769313486231570814527423731704356798070567525844996598917476803157260780*(10**308)}f64']: @@ -598,27 +649,36 @@ class TDTestCase: # * limit set to 1.797693134862316*(10**308) for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16375, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # nchar # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4094, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def tagColIllegalValueCheckCase(self): @@ -629,11 +689,17 @@ class TDTestCase: # bool for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: input_sql1 = self.genFullTypeSql(t0=i)[0] - code = self._conn.insertLines([input_sql1]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql1]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql2 = self.genFullTypeSql(c0=i)[0] - code = self._conn.insertLines([input_sql2]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql2]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # i8 i16 i32 i64 f32 f64 for input_sql in [ @@ -651,8 +717,11 @@ class TDTestCase: self.genFullTypeSql(c6="11.1s45f64")[0], self.genFullTypeSql(c9="1s1u64")[0] ]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check binary and nchar blank stb_name = self.getLongName(7, "letters") @@ -661,18 +730,19 @@ class TDTestCase: input_sql3 = f'{stb_name},t0=t,t1="abc aaa" c0=f 1626006833639000000ns' input_sql4 = f'{stb_name},t0=t,t1=L"abc aaa" c0=f 1626006833639000000ns' for input_sql in [input_sql1, input_sql2, input_sql3, input_sql4]: - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) # check accepted binary and nchar symbols # # * ~!@#$¥%^&*()-+={}|[]、「」:; for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): input_sql1 = f'{stb_name},t0=t c0=f,c1="abc{symbol}aaa" 1626006833639000000ns' input_sql2 = f'{stb_name},t0=t,t1="abc{symbol}aaa" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql1]) - tdSql.checkEqual(code, 0) - code = self._conn.insertLines([input_sql2]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) def duplicateIdTagColInsertCheckCase(self): @@ -681,23 +751,35 @@ class TDTestCase: """ self.cleanStb() input_sql_id = self.genFullTypeSql(id_double_tag=True)[0] - code = self._conn.insertLines([input_sql_id]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_id]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_tag = input_sql.replace("t5", "t6") - code = self._conn.insertLines([input_sql_tag]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_tag]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "c6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "C6") - code = self._conn.insertLines([input_sql_col]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql_col]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) ##### stb exist ##### def noIdStbExistCheckCase(self): @@ -720,8 +802,7 @@ class TDTestCase: self.cleanStb() input_sql, stb_name = self.genFullTypeSql() self.resCmp(input_sql, stb_name) - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) self.resCmp(input_sql, stb_name) def tagColBinaryNcharLengthCheckCase(self): @@ -788,7 +869,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(tb_name1, tb_name2) input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True, ct_add_tag=True) - self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) tb_name3 = self.getNoIdTbName(stb_name) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) @@ -803,29 +884,35 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + self._conn.insert_lines([input_sql]) # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(5, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(6, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError: + pass tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) # # * check col，col+ts max in describe ---> 16143 input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(12, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) + tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(13, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -838,28 +925,32 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) + code = self._conn.insert_lines([input_sql]) # * legal nchar could not be larger than 16374/4 input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(1, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(2, "letters")}" c0=f 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(4, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkEqual(code, 0) + self._conn.insert_lines([input_sql]) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(5, "letters")}" 1626006833639000000ns' - code = self._conn.insertLines([input_sql]) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines([input_sql]) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) @@ -880,8 +971,7 @@ class TDTestCase: "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns", "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - code = self._conn.insertLines(lines) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(lines) def multiInsertCheckCase(self, count): """ @@ -894,8 +984,7 @@ class TDTestCase: for i in range(count): input_sql = self.genFullTypeSql(stb_name=stb_name, t7=f'"{self.getLongName(8, "letters")}"', c7=f'"{self.getLongName(8, "letters")}"', id_noexist_tag=True)[0] sql_list.append(input_sql) - code = self._conn.insertLines(sql_list) - tdSql.checkEqual(code, 0) + self._conn.insert_lines(sql_list) def batchErrorInsertCheckCase(self): """ @@ -905,8 +994,11 @@ class TDTestCase: stb_name = self.getLongName(8, "letters") lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", f"{stb_name},t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"] - code = self._conn.insertLines(lines) - tdSql.checkNotEqual(code, 0) + try: + self._conn.insert_lines(lines) + raise Exception("should not reach here") + except LinesError as err: + tdSql.checkNotEqual(err.errno, 0) def genSqlList(self, count=5, stb_name="", tb_name=""): """ @@ -957,7 +1049,7 @@ class TDTestCase: def genMultiThreadSeq(self, sql_list): tlist = list() for insert_sql in sql_list: - t = threading.Thread(target=self._conn.insertLines,args=([insert_sql[0]],)) + t = threading.Thread(target=self._conn.insert_lines,args=([insert_sql[0]],)) tlist.append(t) return tlist @@ -1155,16 +1247,18 @@ class TDTestCase: def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" input_sql2 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64 c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64 1626006933640000000ns" - code = self._conn.insertLines([input_sql1]) - code = self._conn.insertLines([input_sql2]) - print(code) - # self._conn.insertLines([input_sql2]) + try: + self._conn.insert_lines([input_sql1]) + self._conn.insert_lines([input_sql2]) + except LinesError as err: + print(err.errno) + # self._conn.insert_lines([input_sql2]) # input_sql3 = f'abcd,id="cc¥Ec",t0=True,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="ndsfdrum",t8=L"ncharTagValue" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="igwoehkm",c8=L"ncharColValue",c9=7u64 0' # print(input_sql3) # input_sql4 = 'hmemeb,id="kilrcrldgf",t0=F,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7="fysodjql",t8=L"ncharTagValue" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7="waszbfvc",c8=L"ncharColValue",c9=7u64 0' - # code = self._conn.insertLines([input_sql3]) + # code = self._conn.insert_lines([input_sql3]) # print(code) - # self._conn.insertLines([input_sql4]) + # self._conn.insert_lines([input_sql4]) def runAll(self): self.initCheckCase() @@ -1222,7 +1316,11 @@ class TDTestCase: def run(self): print("running {}".format(__file__)) self.createDb() - self.runAll() + try: + self.runAll() + except Exception as err: + print(''.join(traceback.format_exception(None, err, err.__traceback__))) + raise err # self.tagColIllegalValueCheckCase() # self.test() diff --git a/tests/pytest/query/queryGroupTbname.py b/tests/pytest/query/queryGroupTbname.py new file mode 100644 index 0000000000000000000000000000000000000000..095feed52fa7c44e2dd1b98a5f15c7034165b95e --- /dev/null +++ b/tests/pytest/query/queryGroupTbname.py @@ -0,0 +1,60 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def queryGroupTbname(self): + ''' + select a1,a2...a10 from stb where tbname in (t1,t2,...t10) and ts... + ''' + tdCom.cleanTb() + table_name = tdCom.getLongName(8, "letters_mixed") + tbname_list = list(map(lambda x: f'table_name_sub{x}', range(1, 11))) + tb_str = "" + + for tbname in tbname_list: + globals()[tbname] = tdCom.getLongName(8, "letters_mixed") + tdSql.execute(f'CREATE TABLE {table_name} (ts timestamp, {table_name_sub1} tinyint, \ + {table_name_sub2} smallint, {table_name_sub3} int, {table_name_sub4} bigint, \ + {table_name_sub5} float, {table_name_sub6} double, {table_name_sub7} binary(20),\ + {table_name_sub8} nchar(20), {table_name_sub9} bool) tags ({table_name_sub10} binary(20))') + + for tbname in tbname_list: + tb_str += tbname + tdSql.execute(f'create table {globals()[tbname]} using {table_name} tags ("{globals()[tbname]}")') + + for i in range(10): + for tbname in tbname_list: + tdSql.execute(f'insert into {globals()[tbname]} values (now, 1, 2, 3, 4, 1.1, 2.2, "{globals()[tbname]}", "{globals()[tbname]}", True)') + + for i in range(100): + tdSql.query(f'select {table_name_sub1},{table_name_sub2},{table_name_sub3},{table_name_sub4},{table_name_sub5},{table_name_sub6},{table_name_sub7},{table_name_sub8},{table_name_sub9} from {table_name} where tbname in ("{table_name_sub1}","{table_name_sub2}","{table_name_sub3}","{table_name_sub4}","{table_name_sub5}","{table_name_sub6}","{table_name_sub7}","{table_name_sub8}","{table_name_sub9}") and ts >= "1980-01-01 00:00:00.000"') + tdSql.checkRows(90) + + def run(self): + tdSql.prepare() + self.queryGroupTbname() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/util/sub.py b/tests/pytest/util/sub.py index 2e3c2a96b7312c176c25bd35e109e146e5c4593f..664d830b86290e81e8ac1726f92193380d8c7715 100644 --- a/tests/pytest/util/sub.py +++ b/tests/pytest/util/sub.py @@ -29,9 +29,10 @@ class TDSub: self.sub.close(keepProgress) def consume(self): - self.data = self.sub.consume() - self.consumedRows = len(self.data) - self.consumedCols = len(self.sub.fields) + self.result = self.sub.consume() + self.result.fetch_all() + self.consumedRows = self.result.row_count + self.consumedCols = self.result.field_count return self.consumedRows def checkRows(self, expectRows): diff --git a/tests/script/api/stmt.c b/tests/script/api/stmt.c index 168d281f7062ae892643cd61c53e0448106201b7..baf40c1421df1de4afcc8570288f642df067130a 100644 --- a/tests/script/api/stmt.c +++ b/tests/script/api/stmt.c @@ -465,6 +465,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { TAOS_RES *result = taos_stmt_use_result(stmt); assert(result != NULL); print_result(result); + taos_free_result(result); assert(taos_stmt_close(stmt) == 0); free(params); free(stmt_sql);