From 097c99b661999ada844e0732659bf0a41e57129a Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 27 May 2020 14:22:57 +0800 Subject: [PATCH] [sync master] --- src/client/CMakeLists.txt | 2 +- src/client/inc/tsclient.h | 2 +- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 4 +- src/client/src/TSDBJNIConnector.c | 36 +- src/client/src/tscAsync.c | 29 +- src/client/src/tscFunctionImpl.c | 295 ++++++-- src/client/src/tscLocal.c | 13 + src/client/src/tscParseInsert.c | 13 +- src/client/src/tscProfile.c | 4 +- src/client/src/tscSQLParser.c | 22 +- src/client/src/tscSQLParserImpl.c | 3 + src/client/src/tscSecondaryMerge.c | 43 +- src/client/src/tscServer.c | 54 +- src/client/src/tscSql.c | 11 +- src/client/src/tscStream.c | 24 +- src/client/src/tscSub.c | 3 +- src/client/src/tscUtil.c | 20 +- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/TSDBConnection.java | 8 + .../com/taosdata/jdbc/TSDBJNIConnector.java | 20 +- .../java/com/taosdata/jdbc/TSDBSubscribe.java | 185 +++++ .../taosdata/jdbc/TSDBSubscribeCallBack.java | 19 + .../src/test/java/TestAsyncTSDBSubscribe.java | 83 +++ .../src/test/java/TestPreparedStatement.java | 8 +- .../test/java/TestTSDBResultSetRowData.java | 35 + .../jdbc/src/test/java/TestTSDBSubscribe.java | 83 +++ .../python/linux/python2/taos/cinterface.py | 3 + .../python/linux/python3/taos/cinterface.py | 3 + src/inc/sdb.h | 2 +- src/inc/taosmsg.h | 22 +- src/inc/textbuffer.h | 2 +- src/inc/tinterpolation.h | 4 +- src/inc/tsdb.h | 6 +- src/inc/tsqlfunction.h | 7 + src/inc/tutil.h | 1 + src/kit/shell/CMakeLists.txt | 2 +- src/kit/shell/src/shellEngine.c | 3 +- src/kit/shell/src/shellImport.c | 2 + src/kit/taosdump/taosdump.c | 60 +- src/modules/http/src/gcJson.c | 8 +- src/modules/http/src/httpAuth.c | 2 + src/modules/http/src/restHandle.c | 4 +- src/os/darwin/inc/os.h | 4 +- src/os/windows/CMakeLists.txt | 2 +- src/os/windows/inc/os.h | 7 +- src/os/windows/src/twindows.c | 28 + src/rpc/CMakeLists.txt | 2 +- src/rpc/src/trpc.c | 6 +- src/rpc/src/tstring.c | 2 +- src/rpc/src/ttcpclient.c | 4 + src/rpc/src/ttcpserver.c | 11 + src/sdb/inc/sdbint.h | 2 +- src/sdb/src/sdbEngine.c | 4 +- src/system/detail/inc/mgmt.h | 4 +- src/system/detail/inc/vnode.h | 2 +- src/system/detail/inc/vnodeQueryImpl.h | 7 +- src/system/detail/inc/vnodeRead.h | 10 + src/system/detail/src/mgmtMeter.c | 1 + src/system/detail/src/mgmtShell.c | 57 +- src/system/detail/src/mgmtSupertableQuery.c | 3 +- src/system/detail/src/mgmtUser.c | 10 + src/system/detail/src/vnodeFile.c | 2 +- src/system/detail/src/vnodeQueryImpl.c | 695 +++++++++++++----- src/system/detail/src/vnodeRead.c | 25 +- src/system/detail/src/vnodeStore.c | 2 + src/system/lite/src/mgmtDnodeInt.spec.c | 1 + src/system/lite/src/mgmtShell.spec.c | 5 + src/util/CMakeLists.txt | 2 +- src/util/src/tcache.c | 2 + src/util/src/tcompression.c | 4 +- src/util/src/textbuffer.c | 10 +- src/util/src/tglobalcfg.c | 16 +- src/util/src/tinterpolation.c | 51 +- src/util/src/tnote.c | 4 +- src/util/src/tpercentile.c | 8 +- src/util/src/ttime.c | 2 + src/util/src/ttokenizer.c | 2 +- src/util/src/ttypes.c | 4 +- src/util/src/tutil.c | 17 + src/util/src/version.c | 12 +- .../com/taosdata/generator/DataGenerator.java | 5 +- tests/examples/c/CMakeLists.txt | 2 +- tests/examples/go/src/taosapp/taosapp.go | 2 +- 83 files changed, 1715 insertions(+), 476 deletions(-) create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java create mode 100644 src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java create mode 100644 src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java create mode 100644 src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java create mode 100644 src/connector/jdbc/src/test/java/TestTSDBSubscribe.java diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 92d6b61eb2..d0336a752c 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,7 +31,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "build version ${VERSION_INFO}") SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3e4da10aec..f931bd4729 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -197,7 +197,7 @@ typedef struct SDataBlockList { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint16_t type; // query/insert/import type - char intervalTimeUnit; + char slidingTimeUnit; int64_t etime, stime; int64_t intervalTime; // aggregation time interval diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 8dbe63d75a..d976b2be39 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -142,8 +142,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp * Method: consumeImp * Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData; */ -JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp - (JNIEnv *, jobject, jlong, jint); +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp + (JNIEnv *, jobject, jlong); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 228403c79d..27b28c8f33 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -606,7 +606,7 @@ static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, in return rowobj; } -JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, jint timeout) { +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub); jniGetGlobalMethod(env); @@ -616,38 +616,14 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNI int64_t start = taosGetTimestampMs(); int count = 0; - while (true) { - TAOS_RES * res = taos_consume(tsub); - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return NULL; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - int num_fields = taos_num_fields(res); - while (true) { - TAOS_ROW row = taos_fetch_row(res); - if (row == NULL) { - break; - } - jobject rowobj = convert_one_row(env, row, fields, num_fields); - (*env)->CallBooleanMethod(env, rows, g_arrayListAddFp, rowobj); - count++; - } + TAOS_RES *res = taos_consume(tsub); - if (count > 0) { - break; - } - if (timeout == -1) { - continue; - } - if (((int)(taosGetTimestampMs() - start)) >= timeout) { - jniTrace("jobj:%p, sub:%ld, timeout", jobj, sub); - break; - } + if (res == NULL) { + jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); + return NULL; } - return rows; + return res; } JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, jboolean keepProgress) { diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 9a85fbf8f6..b3c9ef5f21 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -26,6 +26,7 @@ #include "tutil.h" #include "tnote.h" +extern void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -347,8 +348,8 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { (*pSql->fp)(pSql->param, taosres, code); if (shouldFree) { - tscFreeSqlObj(pSql); tscTrace("%p Async sql is automatically freed in async res", pSql); + tscFreeSqlObj(pSql); } } @@ -494,6 +495,11 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) { SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + if(pMeterMetaInfo->pMeterMeta == NULL) { + code = tscGetMeterMeta(pSql, pMeterMetaInfo); + assert(code == TSDB_CODE_SUCCESS); + } + assert(pMeterMetaInfo->pMeterMeta->numOfTags != 0 && pMeterMetaInfo->vnodeIndex >= 0 && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; @@ -504,11 +510,6 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { tscTrace("%p get metricMeta during super table query successfully", pSql); - code = tscGetMeterMeta(pSql, pMeterMetaInfo); - pRes->code = code; - - if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, 0); pRes->code = code; @@ -553,13 +554,27 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { } if (pSql->pStream) { - tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); /* * NOTE: * transfer the sql function for super table query before get meter/metric meta, * since in callback functions, only tscProcessSql(pStream->pSql) is executed! */ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) + && ( pMeterMetaInfo->pMeterMeta == NULL + || pMeterMetaInfo->pMetricMeta == NULL + || pMeterMetaInfo->pMetricMeta->numOfMeters == 0 + || pMeterMetaInfo->pMetricMeta->numOfVnodes == 0)) + || (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) { + tscTrace("%p stream:%p meta is updated, but no table, clear meter meta and set next launch new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); + tscClearMeterMetaInfo(pMeterMetaInfo, false); + tscSetNextLaunchTimer(pSql->pStream, pSql); + return; + } + + tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 9399d188e4..ee0f3dc6d0 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -20,6 +20,7 @@ #include "thistogram.h" #include "tinterpolation.h" #include "tlog.h" +#include "tpercentile.h" #include "tscJoinProcess.h" #include "tscSyntaxtreefunction.h" #include "tscompression.h" @@ -27,7 +28,6 @@ #include "ttime.h" #include "ttypes.h" #include "tutil.h" -#include "tpercentile.h" #define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) #define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) @@ -3669,7 +3669,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcsspn(str, accept); + size_t n = wcscspn(str, accept); str += n; if (str[0] == 0 || (n >= size - 1)) { @@ -3678,7 +3678,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c str++; - int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); + int32_t ret = WCSPatternMatch(&patterStr[i], str, twcslen(str), pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } @@ -4104,8 +4104,6 @@ static void twa_function(SQLFunctionCtx *pCtx) { if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } - - // pCtx->numOfIteratedElems += notNullElems; } static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -4138,7 +4136,6 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { pInfo->lastKey = primaryKey[index]; setTWALastVal(pCtx, pData, 0, pInfo); - // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; if (pResInfo->superTableQ) { @@ -4403,10 +4400,8 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { } } - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - duration = (duration + 500) / 1000; - - double resultVal = ((double)diff) / duration; + double duration = (pRateInfo->lastKey - pRateInfo->firstKey) / 1000.0; + double resultVal = diff / duration; pTrace("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%f lastValue:%f CorrectionValue:%f resultVal:%f", pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); @@ -4447,62 +4442,191 @@ static void rate_function(SQLFunctionCtx *pCtx) { TSKEY *primaryKey = pCtx->ptsList; pTrace("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - pTrace("%p rate_function() index of null data:%d", pCtx, i); - continue; + + if (pCtx->order == TSQL_SO_ASC) { + + #ifdef NOT_EQUINIX + // prev interpolation exists + if (pCtx->prev.key != -1) { + pRateInfo->firstValue = pCtx->prev.data; + pRateInfo->firstKey = pCtx->prev.key; + pCtx->prev.key = -1; // clear the flag + pTrace("%p get prev interpolation for firstValue:%f firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); + + if (-DBL_MAX == pRateInfo->lastValue) { + pRateInfo->lastValue = pCtx->prev.data; + pRateInfo->lastKey = pCtx->prev.key; + } else if (pCtx->prev.data < pRateInfo->lastValue) { + pRateInfo->CorrectionValue += pRateInfo->lastValue; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + + pRateInfo->lastValue = pCtx->prev.data; + pRateInfo->lastKey = pCtx->prev.key; + pTrace("lastValue:%f lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); + } } - - notNullElems++; + #endif - double v = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = (double)GET_INT8_VAL(pData); - break; - case TSDB_DATA_TYPE_SMALLINT: - v = (double)GET_INT16_VAL(pData); - break; - case TSDB_DATA_TYPE_INT: - v = (double)GET_INT32_VAL(pData); - break; - case TSDB_DATA_TYPE_BIGINT: - v = (double)GET_INT64_VAL(pData); - break; - case TSDB_DATA_TYPE_FLOAT: - v = (double)GET_FLOAT_VAL(pData); - break; - case TSDB_DATA_TYPE_DOUBLE: - v = (double)GET_DOUBLE_VAL(pData); - break; - default: - assert(0); + for (int32_t i = 0; i < pCtx->size; ++i) { + char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + pTrace("%p rate_function() index of null data:%d", pCtx, i); + continue; + } + + notNullElems++; + + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = (double)GET_INT8_VAL(pData); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = (double)GET_INT16_VAL(pData); + break; + case TSDB_DATA_TYPE_INT: + v = (double)GET_INT32_VAL(pData); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)GET_INT64_VAL(pData); + break; + case TSDB_DATA_TYPE_FLOAT: + v = (double)GET_FLOAT_VAL(pData); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = (double)GET_DOUBLE_VAL(pData); + break; + default: + assert(0); + } + + if ((-DBL_MAX == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { + pRateInfo->firstValue = v; + pRateInfo->firstKey = primaryKey[i]; + + pTrace("firstValue:%f firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); + } + + if (-DBL_MAX == pRateInfo->lastValue) { + pRateInfo->lastValue = v; + } else if (v < pRateInfo->lastValue) { + pRateInfo->CorrectionValue += pRateInfo->lastValue; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + } + + pRateInfo->lastValue = v; + pRateInfo->lastKey = primaryKey[i]; + pTrace("lastValue:%f lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); } - - if ((-DBL_MAX == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; + + if (!pCtx->hasNull) { + assert(pCtx->size == notNullElems); + } + + #ifdef NOT_EQUINIX + if (pCtx->next.key != -1) { + if (pCtx->next.data < pRateInfo->lastValue) { + pRateInfo->CorrectionValue += pRateInfo->lastValue; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + } + pRateInfo->lastValue = pCtx->next.data; + pRateInfo->lastKey = pCtx->next.key; + pCtx->next.key = -1; + pTrace("%p get next interpolation for lastValue:%f lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); + } + #endif + } else { + #ifdef NOT_EQUINIX + if (pCtx->next.key != -1) { + pRateInfo->lastValue = pCtx->next.data; + pRateInfo->lastKey = pCtx->next.key; + pCtx->next.key = -1; + pTrace("%p get next interpolation for lastValue:%f lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); + + if (-DBL_MAX == pRateInfo->firstValue) { + pRateInfo->firstValue = pCtx->next.data; + pRateInfo->firstKey = pCtx->next.key; + } else if (pCtx->next.data > pRateInfo->firstValue) { + pRateInfo->CorrectionValue += pCtx->next.data; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + pRateInfo->firstValue = pCtx->next.data; + pRateInfo->firstKey = pCtx->next.key; + pTrace("firstValue:%f firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); + } + } + #endif + + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + pTrace("%p rate_function() index of null data:%d", pCtx, i); + continue; + } + + notNullElems++; + + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = (double)GET_INT8_VAL(pData); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = (double)GET_INT16_VAL(pData); + break; + case TSDB_DATA_TYPE_INT: + v = (double)GET_INT32_VAL(pData); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)GET_INT64_VAL(pData); + break; + case TSDB_DATA_TYPE_FLOAT: + v = (double)GET_FLOAT_VAL(pData); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = (double)GET_DOUBLE_VAL(pData); + break; + default: + assert(0); + } + + if ((-DBL_MAX == pRateInfo->lastValue) || (INT64_MIN == pRateInfo->lastKey)) { + pRateInfo->lastValue = v; + pRateInfo->lastKey = primaryKey[i]; + + pTrace("firstValue:%f firstKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); + } + + if (-DBL_MAX == pRateInfo->firstValue) { + pRateInfo->firstValue = v; + } else if (v > pRateInfo->firstValue) { + pRateInfo->CorrectionValue += v; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + } + + pRateInfo->firstValue = v; + pRateInfo->firstKey = primaryKey[i]; pTrace("firstValue:%f firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); } - - if (-DBL_MAX == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->CorrectionValue += pRateInfo->lastValue; - pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + + if (!pCtx->hasNull) { + assert(pCtx->size == notNullElems); } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - pTrace("lastValue:%f lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } + + #ifdef NOT_EQUINIX + if (pCtx->prev.key != -1) { + if (pCtx->prev.data > pRateInfo->firstValue) { + pRateInfo->CorrectionValue += pCtx->prev.data; + pTrace("CorrectionValue:%f", pRateInfo->CorrectionValue); + } + + pRateInfo->firstValue = pCtx->prev.data; + pRateInfo->firstKey = pCtx->prev.key; + pCtx->prev.key = -1; + pTrace("%p get prev interpolation for firstValue:%f firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); + } + #endif + }; SET_VAL(pCtx, notNullElems, 1); @@ -4637,11 +4761,11 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { pTrace("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%f lastValue:%f CorrectionValue:%f hasResult:%d", pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); - if (pRateInfo->hasResult != DATA_SET_FLAG) { + if ((pRateInfo->hasResult != DATA_SET_FLAG) || (INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey)) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); pTrace("rate_finalizer() output result:%f", *(double *)pCtx->aOutputBuf); @@ -4668,7 +4792,17 @@ static void irate_function(SQLFunctionCtx *pCtx) { if (pCtx->size < 1) { return; } - + + #ifdef NOT_EQUINIX + // next interpolation exists + if (pCtx->next.key != -1) { + pRateInfo->lastValue = pCtx->next.data; + pRateInfo->lastKey = pCtx->next.key; + pCtx->next.key = -1; // clear the flag + pTrace("%p irate_function() get next interpolation for lastValue:%f lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); + } + #endif + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { @@ -4702,24 +4836,32 @@ static void irate_function(SQLFunctionCtx *pCtx) { assert(0); } - // TODO: calc once if only call this function once ???? - if ((INT64_MIN == pRateInfo->lastKey) || (-DBL_MAX == pRateInfo->lastValue)) { + if (1 == notNullElems) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - + pTrace("%p irate_function() lastValue:%f lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); continue; } - - if ((INT64_MIN == pRateInfo->firstKey) || (-DBL_MAX == pRateInfo->firstValue)){ - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - pTrace("%p irate_function() firstValue:%f firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); - break; - } + pRateInfo->firstValue = v; + pRateInfo->firstKey = primaryKey[i]; + + pTrace("%p irate_function() firstValue:%f firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); + break; } + #ifdef NOT_EQUINIX + if (pCtx->prev.key != -1) { + if ((INT64_MIN == pRateInfo->firstKey) || (-DBL_MAX == pRateInfo->firstValue)) { + pRateInfo->firstValue = pCtx->prev.data; + pRateInfo->firstKey = pCtx->prev.key; + pCtx->prev.key = -1; + pTrace("%p irate_function() get prev interpolation for firstValue:%f firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); + } + } + #endif + SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -4803,6 +4945,10 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (pInput->hasResult != DATA_SET_FLAG) { continue; } else if (pInput->num == 0) { + if ((INT64_MIN == pInput->lastKey) || (INT64_MIN == pInput->firstKey)) { + continue; + } + pRateInfo->sum += do_calc_rate(pInput); pRateInfo->num++; } else { @@ -4843,6 +4989,11 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { if (pRateInfo->num == 0) { // from meter + if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey)) { + setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + return; + } + *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { *(double*)pCtx->aOutputBuf = pRateInfo->sum; diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7208dc4d72..091eba28a2 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -434,6 +434,19 @@ static void tscProcessServStatus(SSqlObj *pSql) { if (pObj->pHb->res.code == TSDB_CODE_NETWORK_UNAVAIL) { pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL; return; + } else { + int32_t* data = (int32_t*) pObj->pHb->res.data; + if (data != NULL) { + int32_t totalDnode = data[0]; + int32_t onlineDnode = data[1]; + assert(onlineDnode <= totalDnode); + + if (onlineDnode < totalDnode) { + pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL; + return; + } + } + } } else { if (pSql->res.code == TSDB_CODE_NETWORK_UNAVAIL) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index b0d5a58bcf..02a9eec130 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -254,7 +254,9 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, if (pToken->type == TK_NULL) { *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + ((strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0) + || (strncasecmp("nan", pToken->z, pToken->n) == 0) + || (strncasecmp("-nan", pToken->z, pToken->n) == 0))) { *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; } else { double dv; @@ -278,8 +280,10 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, case TSDB_DATA_TYPE_DOUBLE: if (pToken->type == TK_NULL) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + ((strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0) + || (strncasecmp("nan", pToken->z, pToken->n) == 0) + || (strncasecmp("-nan", pToken->z, pToken->n) == 0))) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; } else { double dv; @@ -1434,6 +1438,9 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { pTableDataBlock->size = sizeof(SShellSubmitBlock); pTableDataBlock->rowSize = pMeterMeta->rowSize; + code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows); + if (TSDB_CODE_SUCCESS != code) return -1; + numOfRows += pSql->res.numOfRows; pSql->res.numOfRows = 0; count = 0; diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index a7a774b3a8..ab214384ad 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -292,7 +292,7 @@ void tscKillConnection(STscObj *pObj) { pthread_mutex_unlock(&pObj->mutex); - taos_close(pObj); - tscTrace("connection:%p is killed", pObj); + + taos_close(pObj); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3b4fc0240c..b5c07ba0f4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -282,6 +282,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "name too long"; SCreateDBInfo* pCreateDB = &(pInfo->pDCLInfo->dbOpt); + pCmd->existsCheck = pInfo->pDCLInfo->existsCheck; if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -598,9 +599,6 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { pQueryInfo->intervalTime = pQueryInfo->intervalTime / 1000; } - /* parser has filter the illegal type, no need to check here */ - pQueryInfo->intervalTimeUnit = pQuerySql->interval.z[pQuerySql->interval.n - 1]; - // interval cannot be less than 10 milliseconds if (pQueryInfo->intervalTime < tsMinIntervalTime) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); @@ -689,10 +687,15 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } + + pQueryInfo->slidingTimeUnit = pQuerySql->sliding.z[pQuerySql->sliding.n - 1]; } else { pQueryInfo->slidingTime = pQueryInfo->intervalTime; + + // parser has filter the illegal type, no need to check here + pQueryInfo->slidingTimeUnit = pQuerySql->interval.z[pQuerySql->interval.n - 1]; } - + return TSDB_CODE_SUCCESS; } @@ -1636,13 +1639,16 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt // set the first column ts for diff query if (optr == TK_DIFF) { colIdx += 1; - SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; + SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, 0); insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr); + } else if ((optr >= TK_RATE) && (optr <= TK_AVG_IRATE)) { + SColumnIndex index1 = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnBaseInfoInsert(pQueryInfo, &index1); } // functions can not be applied to tags @@ -1940,6 +1946,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) { columnIndex = i; + break; } } @@ -2588,7 +2595,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType); - size_t len = wcslen((wchar_t*)pColumnFilter->pz); + size_t len = twcslen((wchar_t*)pColumnFilter->pz); pColumnFilter->len = len * TSDB_NCHAR_SIZE; } else { tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType); @@ -4650,6 +4657,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pMeterMetaInfo->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfMeters == 0) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + pSql->res.qhandle = 0x1; // to pass the qhandle check; } // keep original limitation value in globalLimit @@ -5621,7 +5629,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return doLocalQueryProcess(pQueryInfo, pQuerySql); } - if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM) { + if (pQuerySql->from->nExpr > 2) { // not allowed more than 2 table join return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index 17e1c6f457..1bd43fe490 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -446,6 +446,8 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { break; case 'a': break; + case 'u': + return 0; default: { ; return -1; @@ -813,6 +815,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.dbname = *pToken; tTokenListAppend(pInfo->pDCLInfo, pIgExists); + pInfo->pDCLInfo->existsCheck = (pIgExists->n == 1); } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 51a59005f0..d7c0eda746 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -231,12 +231,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd idx += 1; } } - assert(idx >= pReducer->numOfBuffer); + if (idx == 0) { + free(pReducer); return; } - pReducer->numOfBuffer = idx; + pReducer->numOfBuffer = idx; // the actual entries that has result for merge SCompareParam *param = malloc(sizeof(SCompareParam)); param->pLocalData = pReducer->pLocalDataSrc; @@ -324,7 +325,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo; taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, @@ -613,6 +614,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); if (pSchema == NULL) { + tfree(*pMemBuffer); tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; @@ -634,15 +636,27 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + if (pModel == NULL) { + goto _error_memory; + } for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); + + if ((*pMemBuffer)[i] == NULL) { + for (int32_t j=0; j < i; ++j ) { + destroyExtMemBuffer((*pMemBuffer)[j]); + } + goto _error_memory; + } (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - return pRes->code; + for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { + destroyExtMemBuffer((*pMemBuffer)[i]); + } + goto _error_memory; } // final result depends on the fields number @@ -685,6 +699,13 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr tfree(pSchema); return TSDB_CODE_SUCCESS; + +_error_memory: + tfree(pSchema); + tfree(*pMemBuffer); + tscError("%p failed to allocate memory", pSql); + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + return pRes->code; } /** @@ -698,7 +719,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe destroyColumnModel(pFinalModel); tOrderDescDestroy(pDesc); for (int32_t i = 0; i < numOfVnodes; ++i) { - pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); + pMemBuffer[i] = destroyExtMemBuffer(pMemBuffer[i]); } tfree(pMemBuffer); @@ -779,7 +800,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize); @@ -923,7 +944,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo while (1) { int32_t remains = taosNumOfRemainPoints(pInterpoInfo); TSKEY etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->intervalTime, - pQueryInfo->intervalTimeUnit, precision); + pQueryInfo->slidingTimeUnit, precision); int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->intervalTime, etime, pLocalReducer->resColModel->capacity); @@ -1275,7 +1296,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t newTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, precision); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime, pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize); @@ -1305,7 +1326,7 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) { int32_t remain = taosNumOfRemainPoints(pInterpoInfo); TSKEY ekey = - taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, p); + taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, p); int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain, pQueryInfo->intervalTime, ekey, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo @@ -1338,7 +1359,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, - pQueryInfo->intervalTimeUnit, precision); + pQueryInfo->slidingTimeUnit, precision); int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->intervalTime, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d2091e45ac..ea42f32cf6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -106,12 +106,12 @@ static int32_t tscGetMgmtConnMaxRetryTimes() { return tscMgmtIpList.numOfIps * factor; } -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { +int32_t tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; + if (pObj == NULL) return TSDB_CODE_APP_ERROR; if (pObj != pObj->signature) { tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; + return TSDB_CODE_APP_ERROR; } SSqlObj *pSql = pObj->pHb; @@ -128,11 +128,19 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->queryId) tscKillQuery(pObj, pRsp->queryId); if (pRsp->streamId) tscKillStream(pObj, pRsp->streamId); } + + if (pRes->data == NULL) { + pRes->data = calloc(2, sizeof(int32_t)); + } + + ((int32_t*)pRes->data)[0] = htonl(pRsp->totalDnodes); + ((int32_t*)pRes->data)[1] = htonl(pRsp->onlineDnodes); } else { tscTrace("heart beat failed, code:%d", code); } taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); + return code; } void tscProcessActivityTimer(void *handle, void *tmrId) { @@ -398,9 +406,9 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { if (pSql->freed || pObj->signature != pObj) { tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed, pObj, pObj->signature); - taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); + //taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); tscFreeSqlObj(pSql); - return ahandle; + return NULL; } SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); @@ -600,8 +608,8 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { taos_close(pObj); tscTrace("%p Async sql close failed connection", pSql); } else { - tscFreeSqlObj(pSql); tscTrace("%p Async sql is automatically freed", pSql); + tscFreeSqlObj(pSql); } } } @@ -710,15 +718,21 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; void *asyncFp = pSql->fp; if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_METRIC) { - tscBuildMsg[pCmd->command](pSql, NULL); + code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + return code; } - int32_t code = tscSendMsgToServer(pSql); + code = tscSendMsgToServer(pSql); if (asyncFp) { if (code != TSDB_CODE_SUCCESS) { @@ -994,7 +1008,13 @@ int tscLaunchSTableSubqueries(SSqlObj *pSql) { SRetrieveSupport* pSupport = pSub->param; tscTrace("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); - tscProcessSql(pSub); + int code = tscProcessSql(pSub); + if (code != TSDB_CODE_SUCCESS) { + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + doCleanupSubqueries(pSql, i, pState); + pRes->code = code; + return pRes->code; + } } return TSDB_CODE_SUCCESS; @@ -1681,7 +1701,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); - pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit; + pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); if (pQueryInfo->intervalTime < 0) { @@ -2148,7 +2168,12 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(SDropUserMsg); pCmd->payloadLen = pMsg - pStart; - pCmd->msgType = TSDB_MSG_TYPE_DROP_USER; + + if (pInfo->type == TSDB_SQL_DROP_ACCT) { + pCmd->msgType = TSDB_MSG_TYPE_DROP_ACCT; + } else { + pCmd->msgType = TSDB_MSG_TYPE_DROP_USER; + } return TSDB_CODE_SUCCESS; } @@ -2697,7 +2722,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscTrace("%p build load multi-metermeta msg completed, numOfMeters:%d, msg size:%d", pSql, pCmd->count, pCmd->payloadLen); - return pCmd->payloadLen; + return TSDB_CODE_SUCCESS; } static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { @@ -2924,7 +2949,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessMeterMetaRsp(SSqlObj *pSql) { @@ -3684,7 +3709,8 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { SMeterMetaInfo *pMMInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMMInfo->name); + SMeterMeta *pMeterMeta = (SMeterMeta *)taosGetDataFromExists(tscCacheHandle, pQueryInfo->pMeterInfo[i]->pMeterMeta); + assert(pMeterMeta != NULL); tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pMeterMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 99bc9632de..261484c712 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -321,6 +321,9 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + if(NULL == pQueryInfo){ + return NULL; + } return pQueryInfo->fieldsInfo.pFields; } @@ -726,6 +729,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { +#if 0 SSqlObj *pSql = (SSqlObj *)res; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -765,6 +769,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } return nRows; +#endif + + (*rows) = taos_fetch_row(res); + return ((*rows) != NULL)? 1:0; + } int taos_select_db(TAOS *taos, const char *db) { @@ -796,8 +805,8 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); if (pSql->fp != NULL) { pSql->thandle = NULL; - tscFreeSqlObj(pSql); tscTrace("%p Async SqlObj is freed by app", pSql); + tscFreeSqlObj(pSql); } else if (keepCmd) { tscFreeSqlResult(pSql); } else { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 8dc8ad49b5..804fb99c86 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -28,7 +28,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows); static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows); -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); +void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer); static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) { @@ -97,6 +97,18 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { return; } + if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) + && ( pMeterMetaInfo->pMeterMeta == NULL + || pMeterMetaInfo->pMetricMeta == NULL + || pMeterMetaInfo->pMetricMeta->numOfMeters == 0 + || pMeterMetaInfo->pMetricMeta->numOfVnodes == 0)) + || (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) { + tscTrace("%p no table in metricmeta, no launch query", pSql); + tscClearMeterMetaInfo(pMeterMetaInfo, false); + tscSetNextLaunchTimer(pStream, pSql); + return; + } + tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pMeterMetaInfo->name); tscProcessSql(pStream->pSql); @@ -323,7 +335,7 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { } -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { +void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { int64_t timer = 0; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); @@ -374,8 +386,7 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { } static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { - int64_t minIntervalTime = - (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinIntervalTime * 1000L : tsMinIntervalTime; + int64_t minIntervalTime = tsMinIntervalTime; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); @@ -391,8 +402,7 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { pQueryInfo->slidingTime = pQueryInfo->intervalTime; } - int64_t minSlidingTime = - (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime; + int64_t minSlidingTime = tsMinSlidingTime; if (pQueryInfo->slidingTime == -1) { pQueryInfo->slidingTime = pQueryInfo->intervalTime; @@ -582,10 +592,10 @@ void taos_close_stream(TAOS_STREAM *handle) { tscRemoveFromStreamList(pStream, pSql); taosTmrStopA(&(pStream->pTimer)); + tscTrace("%p stream:%p is closed", pSql, pStream); tscFreeSqlObj(pSql); pStream->pSql = NULL; - tscTrace("%p stream:%p is closed", pSql, pStream); tfree(pStream); } } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 610c119e6d..b9690e1888 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -104,6 +104,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* return NULL; } + char* sqlstr = NULL; SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -114,7 +115,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* pSql->signature = pSql; pSql->pTscObj = pObj; - char* sqlstr = (char*)malloc(strlen(sql) + 1); + sqlstr = (char*)malloc(strlen(sql) + 1); if (sqlstr == NULL) { tscError("failed to allocate sql string for subscription"); goto failed; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1073eec6ea..1611c5b724 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -87,6 +87,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { MD5Update(&ctx, (uint8_t*)tmp, keyLen); char* pStr = base64_encode(ctx.digest, tListLen(ctx.digest)); strcpy(str, pStr); + free(pStr); } free(tmp); @@ -456,9 +457,11 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { pCmd->command = 0; // pSql->sqlstr will be used by tscBuildQueryStreamDesc - pthread_mutex_lock(&pObj->mutex); - tfree(pSql->sqlstr); - pthread_mutex_unlock(&pObj->mutex); + if (pObj->signature == pObj) { + pthread_mutex_lock(&pObj->mutex); + tfree(pSql->sqlstr); + pthread_mutex_unlock(&pObj->mutex); + } tscFreeSqlResult(pSql); tfree(pSql->pSubs); @@ -2009,7 +2012,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } // create the fields info from the sql functions - SColumnList columnList = {.num = 1}; + SColumnList columnList = {.num = 0}; for(int32_t k = 0; k < numOfOutputCols; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, indexList[k]); @@ -2063,7 +2066,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pMeterMetaInfo->tagColumnIndex); } - assert(pFinalInfo->pMeterMeta != NULL && pNewQueryInfo->numOfTables == 1); + if (pFinalInfo->pMeterMeta == NULL) { + tscError("%p new subquery failed for get pMeterMeta is NULL from cache", pSql); + tscFreeSqlObj(pNew); + return NULL; + } + + assert(pNewQueryInfo->numOfTables == 1); + if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { assert(pFinalInfo->pMetricMeta != NULL); } diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 36f1a1010c..1db4ca878b 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -37,7 +37,7 @@ UTF-8 - 1.8 + 1.7 3.6.0 diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 4640f6b446..062cb63cfd 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -84,6 +84,14 @@ public class TSDBConnection implements Connection { } } + public TSDBSubscribe createSubscribe() throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBSubscribe(this.connector); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + public PreparedStatement prepareStatement(String sql) throws SQLException { if (!this.connector.isClosed()) { return new TSDBPreparedStatement(this.connector, sql); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 3adb601822..ed428ea15f 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -22,8 +22,8 @@ public class TSDBJNIConnector { static volatile Boolean isInitialized = false; static { - System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); + System.loadLibrary("taos"); } /** @@ -261,31 +261,31 @@ public class TSDBJNIConnector { /** * Subscribe to a table in TSDB */ - public long subscribe(String host, String user, String password, String database, String table, long time, int period) { - return subscribeImp(host, user, password, database, table, time, period); + public long subscribe(String topic, String sql, boolean restart, int period) { + return subscribeImp(this.taos, restart, topic, sql, period); } - private native long subscribeImp(String host, String user, String password, String database, String table, long time, int period); + public native long subscribeImp(long connection, boolean restart, String topic, String sql, int period); /** * Consume a subscribed table */ - public TSDBResultSetRowData consume(long subscription) { + public long consume(long subscription) { return this.consumeImp(subscription); } - private native TSDBResultSetRowData consumeImp(long subscription); + private native long consumeImp(long subscription); /** * Unsubscribe a table * * @param subscription */ - public void unsubscribe(long subscription) { - unsubscribeImp(subscription); + public void unsubscribe(long subscription, boolean isKeep) { + unsubscribeImp(subscription, isKeep); } - private native void unsubscribeImp(long subscription); + private native void unsubscribeImp(long subscription, boolean isKeep); /** * Validate if a create table sql statement is correct without actually creating that table @@ -293,7 +293,7 @@ public class TSDBJNIConnector { public boolean validateCreateTableSql(String sql) { long connection = taos; int res = validateCreateTableSqlImp(connection, sql.getBytes()); - return res != 0 ? false : true; + return res == 0; } private native int validateCreateTableSqlImp(long connection, byte[] sqlBytes); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java new file mode 100644 index 0000000000..3b479aafc3 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -0,0 +1,185 @@ +/*************************************************************************** + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + *****************************************************************************/ +package com.taosdata.jdbc; + +import javax.management.OperationsException; +import java.sql.SQLException; +import java.util.Map; +import java.util.TimerTask; +import java.util.concurrent.*; + +public class TSDBSubscribe { + private TSDBJNIConnector connecter = null; + private static ScheduledExecutorService pool; + private static Map timerTaskMap = new ConcurrentHashMap<>(); + private static Map scheduledMap = new ConcurrentHashMap(); + + private static class TimerInstance { + private static final ScheduledExecutorService instance = Executors.newScheduledThreadPool(1); + } + + public static ScheduledExecutorService getTimerInstance() { + return TimerInstance.instance; + } + + public TSDBSubscribe(TSDBJNIConnector connecter) throws SQLException { + if (null != connecter) { + this.connecter = connecter; + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + /** + * sync subscribe + * + * @param topic + * @param sql + * @param restart + * @param period + * @throws SQLException + */ + public long subscribe(String topic, String sql, boolean restart, int period) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + if (period < 1000) { + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.INVALID_VARIABLES)); + } + return this.connecter.subscribe(topic, sql, restart, period); + } + + /** + * async subscribe + * + * @param topic + * @param sql + * @param restart + * @param period + * @param callBack + * @throws SQLException + */ + public long subscribe(String topic, String sql, boolean restart, int period, TSDBSubscribeCallBack callBack) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + final long subscription = this.connecter.subscribe(topic, sql, restart, period); + if (null != callBack) { + pool = getTimerInstance(); + + TSDBTimerTask timerTask = new TSDBTimerTask(subscription, callBack); + + timerTaskMap.put(subscription, timerTask); + + ScheduledFuture scheduledFuture = pool.scheduleAtFixedRate(timerTask, 1, 1000, TimeUnit.MILLISECONDS); + scheduledMap.put(subscription, scheduledFuture); + } + return subscription; + } + + public TSDBResultSet consume(long subscription) throws OperationsException, SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + if (0 == subscription) { + throw new OperationsException("Invalid use of consume"); + } + long resultSetPointer = this.connecter.consume(subscription); + + if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { + return null; + } else { + return new TSDBResultSet(this.connecter, resultSetPointer); + } + } + + /** + * cancel subscribe + * + * @param subscription + * @param isKeep + * @throws SQLException + */ + public void unsubscribe(long subscription, boolean isKeep) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + + if (null != timerTaskMap.get(subscription)) { + synchronized (timerTaskMap.get(subscription)) { + while (1 == timerTaskMap.get(subscription).getState()) { + try { + Thread.sleep(10); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + timerTaskMap.get(subscription).setState(2); + if (!timerTaskMap.isEmpty() && timerTaskMap.containsKey(subscription)) { + timerTaskMap.get(subscription).cancel(); + timerTaskMap.remove(subscription); + scheduledMap.get(subscription).cancel(false); + scheduledMap.remove(subscription); + } + this.connecter.unsubscribe(subscription, isKeep); + } + } else { + this.connecter.unsubscribe(subscription, isKeep); + } + } + + class TSDBTimerTask extends TimerTask { + private long subscription; + private TSDBSubscribeCallBack callBack; + // 0: not running 1: running 2: cancel + private int state = 0; + + public TSDBTimerTask(long subscription, TSDBSubscribeCallBack callBack) { + this.subscription = subscription; + this.callBack = callBack; + } + + public int getState() { + return this.state; + } + + public void setState(int state) { + this.state = state; + } + + @Override + public void run() { + synchronized (this) { + if (2 == state) { + return; + } + + state = 1; + + try { + TSDBResultSet resultSet = consume(subscription); + callBack.invoke(resultSet); + } catch (Exception e) { + this.cancel(); + throw new RuntimeException(e); + } + state = 0; + } + } + } +} + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java new file mode 100644 index 0000000000..c1b9b02fa8 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java @@ -0,0 +1,19 @@ +/*************************************************************************** + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + *****************************************************************************/ +package com.taosdata.jdbc; + +public interface TSDBSubscribeCallBack { + void invoke(TSDBResultSet resultSet); +} diff --git a/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java new file mode 100644 index 0000000000..5b2b6367ec --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java @@ -0,0 +1,83 @@ +import com.taosdata.jdbc.*; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestAsyncTSDBSubscribe { + public static void main(String[] args) { + String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName -topic topicName " + + "-tname tableName -h host"; + if (args.length < 2) { + System.err.println(usage); + return; + } + + String dbName = ""; + String tName = ""; + String host = "localhost"; + String topic = ""; + for (int i = 0; i < args.length; i++) { + if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { + dbName = args[++i]; + } + if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tName = args[++i]; + } + if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + } + if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) { + System.err.println(usage); + return; + } + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first")); + long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second")); + int a = 0; + Thread.sleep(2000); + subscribe.unsubscribe(subscribId, true); + System.err.println("cancel subscribe"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static class CallBack implements TSDBSubscribeCallBack { + private String name = ""; + + public CallBack(String name) { + this.name = name; + } + + @Override + public void invoke(TSDBResultSet resultSet) { + try { + while (null !=resultSet && resultSet.next()) { + System.out.print("callback_" + name + ": "); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resultSet.getString(i) + "\t"); + } + System.out.println(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java index 2e2cc0ede6..fce2bd1428 100644 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -11,8 +11,14 @@ public class TestPreparedStatement { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.117"); - Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.117:0/?user=root&password=taosdata", properties); + Connection connection = DriverManager.getConnection("jdbc:TAOS://10.211.55.3:0/log?user=root&password=taosdata", properties); + String createSql = "create table t (ts timestamp, speed int);"; + Statement statement = connection.createStatement(); + statement.executeQuery(createSql); String rawSql = "SELECT ts, c1 FROM (select c1, ts from db.tb1) SUB_QRY"; + if (1 < 2) { + return; + } // String[] params = new String[]{"ts", "c1"}; PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); ResultSet resSet = pstmt.executeQuery(); diff --git a/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java b/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java new file mode 100644 index 0000000000..62d4c2ff61 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java @@ -0,0 +1,35 @@ +import com.taosdata.jdbc.ColumnMetaData; +import com.taosdata.jdbc.DatabaseMetaDataResultSet; +import com.taosdata.jdbc.TSDBResultSetRowData; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class TestTSDBResultSetRowData { + public static void main(String[] args) throws SQLException { + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + List columnMetaDataList = new ArrayList(1); + ColumnMetaData colMetaData = new ColumnMetaData(); + colMetaData.setColIndex(0); + colMetaData.setColName("TABLE_TYPE"); + colMetaData.setColSize(10); + colMetaData.setColType(8); + columnMetaDataList.add(colMetaData); + + List rowDataList = new ArrayList(2); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(2); + rowData.setString(0, "TABLE"); + rowDataList.add(rowData); + rowData = new TSDBResultSetRowData(2); + rowData.setString(0, "STABLE"); + rowDataList.add(rowData); + + resultSet.setColumnMetaDataList(columnMetaDataList); + resultSet.setRowDataList(rowDataList); + + while (resultSet.next()) { + System.out.println(resultSet.getString(1)); + } + } +} diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java new file mode 100644 index 0000000000..f12924c8a6 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java @@ -0,0 +1,83 @@ +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestTSDBSubscribe { + public static void main(String[] args) throws Exception { + String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName " + + "-topic topicName -tname tableName -h host"; + if (args.length < 2) { + System.err.println(usage); + return; + } + + String dbName = ""; + String tName = ""; + String host = "localhost"; + String topic = ""; + for (int i = 0; i < args.length; i++) { + if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { + dbName = args[++i]; + } + if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tName = args[++i]; + } + if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + } + if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) { + System.err.println(usage); + return; + } + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata" + , properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000); + int a = 0; + while (true) { + Thread.sleep(900); + TSDBResultSet resSet = subscribe.consume(subscribId); + + while (resSet.next()) { + for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resSet.getString(i) + "\t"); + } + System.out.println("\n======" + a + "=========="); + } + + a++; + if (a >= 10) { + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (null != subscribe && 0 != subscribId) { + subscribe.unsubscribe(subscribId, true); + } + if (null != connection) { + connection.close(); + } + } + } +} diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 505619436c..d9460efb21 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 7fcedc9fe9..77001609b6 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/inc/sdb.h b/src/inc/sdb.h index a0e0a1b2f2..389aecfb7b 100644 --- a/src/inc/sdb.h +++ b/src/inc/sdb.h @@ -105,7 +105,7 @@ extern SSdbPeer *sdbPeer[]; #endif -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, +void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, char keyType, char *directory, void *(*appTool)(char, void *, char *, int, int *)); void *sdbGetRow(void *handle, void *key); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 4307e41b0b..48ee48eae3 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -439,9 +439,15 @@ typedef struct SSqlFuncExprMsg { } SSqlFuncExprMsg; typedef struct SSqlBinaryExprInfo { - struct tSQLBinaryExpr *pBinExpr; /* for binary expression */ + union { + struct tSQLBinaryExpr *pBinExpr; /* for binary expression */ + int64_t resvSpace0; + }; int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndexEx * pReqColumns; /* source column list */ + union { + SColIndexEx * pReqColumns; /* source column list */ + int64_t resvSpace1; + }; } SSqlBinaryExprInfo; typedef struct SSqlFunctionExpr { @@ -481,8 +487,12 @@ typedef struct SColumnInfo { int16_t colId; int16_t type; int16_t bytes; - int16_t numOfFilters; - SColumnFilterInfo *filters; + int16_t numOfFilters; + union { + SColumnFilterInfo *filters; + int64_t resvSpace; + }; + } SColumnInfo; /* @@ -513,7 +523,7 @@ typedef struct { int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode - char intervalTimeUnit; // time interval type, for revisement of interval(1d) + char slidingTimeUnit; // time interval type, for revisement of interval(1d) int64_t intervalTime; // time interval for aggregation, in million second int64_t slidingTime; // value for sliding window @@ -811,6 +821,8 @@ typedef struct { typedef struct { uint32_t queryId; uint32_t streamId; + uint32_t totalDnodes; + uint32_t onlineDnodes; char killConnection; SIpList ipList; } SHeartBeatRsp; diff --git a/src/inc/textbuffer.h b/src/inc/textbuffer.h index b46b98ed38..ec7a192fad 100644 --- a/src/inc/textbuffer.h +++ b/src/inc/textbuffer.h @@ -136,7 +136,7 @@ tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnMo * @param pMemBuffer * @return */ -void *destoryExtMemBuffer(tExtMemBuffer *pMemBuffer); +void *destroyExtMemBuffer(tExtMemBuffer *pMemBuffer); /** * @param pMemBuffer diff --git a/src/inc/tinterpolation.h b/src/inc/tinterpolation.h index f4b327bcbe..6c0f6f921e 100644 --- a/src/inc/tinterpolation.h +++ b/src/inc/tinterpolation.h @@ -30,7 +30,7 @@ typedef struct SInterpolationInfo { char * prevValues; // previous row of data char * nextValues; // next row of data int32_t numOfTags; - char ** pTags; // tags value for current interoplation + char ** pTags; // tags value for current interpolation } SInterpolationInfo; typedef struct SPoint { @@ -83,6 +83,8 @@ int32_t taosDoInterpoResult(SInterpolationInfo *pInterpoInfo, int16_t interpoTyp int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); +int taosDoLinearInterpolationD(int32_t type, SPoint* point1, SPoint* point2, SPoint* point); + #ifdef __cplusplus } #endif diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index aa712ab622..9d6c2ef16b 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -103,8 +103,8 @@ extern "C" { #define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 6mb #define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 16 -#define TSDB_MAX_TAGS_LEN 512 -#define TSDB_MAX_TAGS 32 +#define TSDB_MAX_TAGS_LEN 2048 +#define TSDB_MAX_TAGS 128 #define TSDB_AUTH_LEN 16 #define TSDB_KEY_LEN 16 @@ -133,7 +133,7 @@ extern "C" { #define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) -#define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size +#define TSDB_DEFAULT_PAYLOAD_SIZE 4096 // default payload size #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_SQLCMD_SIZE 1024 #define TSDB_MAX_VNODES 256 diff --git a/src/inc/tsqlfunction.h b/src/inc/tsqlfunction.h index b423589672..b2e53a931b 100644 --- a/src/inc/tsqlfunction.h +++ b/src/inc/tsqlfunction.h @@ -167,6 +167,11 @@ typedef struct SExtTagsInfo { struct SQLFunctionCtx **pTagCtxList; } SExtTagsInfo; +typedef struct SBoundaryData { + TSKEY key; + double data; +} SBoundaryData; + // sql function runtime context typedef struct SQLFunctionCtx { int32_t startOffset; @@ -195,6 +200,8 @@ typedef struct SQLFunctionCtx { SResultInfo *resultInfo; SExtTagsInfo tagInfo; + SBoundaryData prev; // this value may be less or equalled to the start time of time window + SBoundaryData next; // this value may be greater or equalled to the end time of time window } SQLFunctionCtx; typedef struct SQLAggFuncElem { diff --git a/src/inc/tutil.h b/src/inc/tutil.h index b66da28697..2333d8f6b3 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -130,6 +130,7 @@ extern "C" { #define POW2(x) ((x) * (x)) +size_t twcslen(const wchar_t *wcs); int32_t strdequote(char *src); void strtrim(char *src); diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 1845078b69..d8b4d1ad46 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -19,7 +19,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME ${DB_CLIENT_NAME}) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) LIST(APPEND SRC ./src/shellEngine.c) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 1d530ea0b8..0608663942 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -818,7 +818,8 @@ void shellGetGrantInfo(void *con) { if (code == TSDB_CODE_OPS_NOT_SUPPORT) { fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con)); } else { - fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + //fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + fprintf(stdout, "Server is Enterprise Edition, version is %s\n\n", taos_get_server_info(con)); } return; } diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index dd04f935e7..0d0e7e9855 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -141,6 +141,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); + free(cmd); return; } @@ -166,6 +167,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { if (f == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); + free(cmd); return; } diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 46d82e43e4..93035cd3de 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "taos.h" #include "taosmsg.h" @@ -162,6 +163,7 @@ static struct argp_option options[] = { {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is "DB_COMPANY".", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, @@ -189,6 +191,7 @@ struct arguments { char *password; uint16_t port; char cversion[TSDB_FILENAME_LEN+1]; + uint16_t mysqlFlag; // output file char outpath[TSDB_FILENAME_LEN+1]; char inpath[TSDB_FILENAME_LEN+1]; @@ -236,6 +239,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'P': arguments->port = atoi(arg); break; + case 'q': + arguments->mysqlFlag = atoi(arg); + break; case 'v': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid client vesion %s\n", arg); @@ -325,6 +331,7 @@ int taosDumpOut(struct arguments *arguments); int taosDumpIn(struct arguments *arguments); void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon); void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp); void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp); int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon); @@ -340,6 +347,7 @@ struct arguments tsArguments = { DB_COMPANY, 0, "", + 0, // outpath and inpath "", "", @@ -383,7 +391,8 @@ int main(int argc, char *argv[]) { printf("user: %s\n", tsArguments.user); printf("password: %s\n", tsArguments.password); printf("port: %u\n", tsArguments.port); - printf("cversion: %s\n", tsArguments.cversion); + printf("cversion: %s\n", tsArguments.cversion); + printf("mysqlFlag: %d", tsArguments.mysqlFlag); printf("outpath: %s\n", tsArguments.outpath); printf("inpath: %s\n", tsArguments.inpath); printf("encode: %s\n", tsArguments.encode); @@ -616,7 +625,7 @@ int taosDumpOut(struct arguments *arguments) { int32_t count = 0; STableRecordInfo tableRecordInfo; - char tmpBuf[TSDB_FILENAME_LEN+1] = {0}; + char tmpBuf[TSDB_FILENAME_LEN+9] = {0}; if (arguments->outpath[0] != 0) { sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath); } else { @@ -917,6 +926,8 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp); } + pstr += sprintf(pstr, ";"); + fprintf(fp, "%s\n\n", tmpCommand); free(tmpCommand); } @@ -927,7 +938,7 @@ void* taosDumpOutWorkThreadFp(void *arg) STableRecord tableRecord; int fd; - char tmpFileName[TSDB_FILENAME_LEN + 1] = {0}; + char tmpFileName[TSDB_FILENAME_LEN + 128] = {0}; sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex); fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { @@ -936,7 +947,7 @@ void* taosDumpOutWorkThreadFp(void *arg) } FILE *fp = NULL; - memset(tmpFileName, 0, TSDB_FILENAME_LEN); + memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128); if (tsArguments.outpath[0] != 0) { sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); @@ -1235,7 +1246,7 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { } } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); fprintf(fp, "%s\n", tmpBuf); @@ -1288,7 +1299,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols /* } */ } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); fprintf(fp, "%s\n", tmpBuf); free(tmpBuf); @@ -1358,14 +1369,31 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* return -1; } + char sqlStr[8] = "\0"; + if (arguments->mysqlFlag) { + sprintf(sqlStr, "INSERT"); + } else { + sprintf(sqlStr, "IMPORT"); + } + + int rowFlag = 0; count = 0; while ((row = taos_fetch_row(tmpResult)) != NULL) { pstr = tmpBuffer; if (count == 0) { - pstr += sprintf(pstr, "IMPORT INTO %s VALUES (", tbname); - } else { - pstr += sprintf(pstr, "("); + pstr += sprintf(pstr, "%s INTO %s VALUES (", sqlStr, tbname); + } else { + if (arguments->mysqlFlag) { + if (0 == rowFlag) { + pstr += sprintf(pstr, "("); + rowFlag++; + } else { + pstr += sprintf(pstr, ", ("); + } + } else { + pstr += sprintf(pstr, "("); + } } for (int col = 0; col < numFields; col++) { @@ -1409,12 +1437,22 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* pstr += sprintf(pstr, "\'%s\'", tbuf); break; case TSDB_DATA_TYPE_TIMESTAMP: - pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]); + if (!arguments->mysqlFlag) { + pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[col]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + pstr += sprintf(pstr, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } break; default: break; } } + pstr += sprintf(pstr, ") "); totalRows++; @@ -1422,7 +1460,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* fprintf(fp, "%s", tmpBuffer); if (count >= arguments->data_batch) { - fprintf(fp, "\n"); + fprintf(fp, ";\n"); count = 0; } //else { //fprintf(fp, "\\\n"); diff --git a/src/modules/http/src/gcJson.c b/src/modules/http/src/gcJson.c index 51f7f18345..66b4e3dfe0 100644 --- a/src/modules/http/src/gcJson.c +++ b/src/modules/http/src/gcJson.c @@ -152,9 +152,11 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name); - memcpy(target + len, (char *) row[i], fields[i].bytes); - len = strlen(target); + if (row[i]!= NULL){ + len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name); + memcpy(target + len, (char *) row[i], fields[i].bytes); + len = strlen(target); + } break; default: len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, "-"); diff --git a/src/modules/http/src/httpAuth.c b/src/modules/http/src/httpAuth.c index 3aea57b504..a5f043adc6 100644 --- a/src/modules/http/src/httpAuth.c +++ b/src/modules/http/src/httpAuth.c @@ -75,6 +75,8 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) { unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, ip:%s, taosd token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token); + if (base64) + free(base64); return false; } if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) { diff --git a/src/modules/http/src/restHandle.c b/src/modules/http/src/restHandle.c index a307700866..57e34affed 100644 --- a/src/modules/http/src/restHandle.c +++ b/src/modules/http/src/restHandle.c @@ -68,9 +68,7 @@ bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) { } - /* - * for async test - * / + // for async test /* if (httpCheckUsedbSql(sql)) { httpSendErrorResp(pContext, HTTP_NO_EXEC_USEDB); diff --git a/src/os/darwin/inc/os.h b/src/os/darwin/inc/os.h index 1aececeec9..a58885a644 100644 --- a/src/os/darwin/inc/os.h +++ b/src/os/darwin/inc/os.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_PLATFORM_LINUX_H -#define TDENGINE_PLATFORM_LINUX_H +#ifndef TDENGINE_PLATFORM_DARWIN_H +#define TDENGINE_PLATFORM_DARWIN_H #ifdef __cplusplus extern "C" { diff --git a/src/os/windows/CMakeLists.txt b/src/os/windows/CMakeLists.txt index dc60b736ea..6e8695a8a8 100644 --- a/src/os/windows/CMakeLists.txt +++ b/src/os/windows/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) diff --git a/src/os/windows/inc/os.h b/src/os/windows/inc/os.h index 3f957e8abd..c26d1cd540 100644 --- a/src/os/windows/inc/os.h +++ b/src/os/windows/inc/os.h @@ -142,8 +142,11 @@ extern "C" { #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) -#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - +#if (_MSC_VER == 1800) + #define atomic_exchange_ptr(ptr, val) InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) +#else + #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) +#endif #ifdef _TD_GO_DLL_ #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap #else diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index f7d23110d8..67adb86e73 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -93,7 +93,11 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { } __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { +#ifdef _WIN64 return _InterlockedExchangeAdd64(ptr, val) + val; +#else + return _InterlockedExchangeAdd(ptr, val) + val; +#endif } // and @@ -377,9 +381,29 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { return writeLen; } +unsigned char _MyBitScanForward64(unsigned long *ret, uint64_t x) { + unsigned long x0 = (unsigned long)x, top, bottom; + _BitScanForward(&top, (unsigned long)(x >> 32)); + _BitScanForward(&bottom, x0); + *ret = x0 ? bottom : 32 + top; + return x != 0; +} + +unsigned char _MyBitScanReverse64(unsigned long *ret, uint64_t x) { + unsigned long x1 = (unsigned long)(x >> 32), top, bottom; + _BitScanReverse(&top, x1); + _BitScanReverse(&bottom, (unsigned long)x); + *ret = x1 ? top + 32 : bottom; + return x != 0; +} + int32_t BUILDIN_CLZL(uint64_t val) { unsigned long r = 0; +#ifdef _WIN64 _BitScanReverse64(&r, val); +#else + _MyBitScanReverse64(&r, val); +#endif return (int)(r >> 3); } @@ -391,7 +415,11 @@ int32_t BUILDIN_CLZ(uint32_t val) { int32_t BUILDIN_CTZL(uint64_t val) { unsigned long r = 0; +#ifdef _WIN64 _BitScanForward64(&r, val); +#else + _MyBitScanForward64(&r, val); +#endif return (int)(r >> 3); } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 8d54d5fda5..e06b462d2c 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -12,7 +12,7 @@ ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/trpc.c) LIST(APPEND SRC ./src/tstring.c) LIST(APPEND SRC ./src/tudp.c) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) LIST(APPEND SRC ./src/thaship.c) LIST(APPEND SRC ./src/trpc.c) diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index d8f0cac874..c8f58db1fe 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -1184,9 +1184,9 @@ void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, uint16_t por taosTmrReset(taosProcessIdleTimer, pServer->idleTime, pConn, pChann->tmrCtrl, &pConn->pIdleTimer); } - if (code == TSDB_CODE_ALREADY_PROCESSED) { - tTrace("%s cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label, - chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId), + if (code == TSDB_CODE_ALREADY_PROCESSED || code == TSDB_CODE_LAST_SESSION_NOT_FINISHED) { + tTrace("%s code:%d, cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label, + code, chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId), pHeader->tranId, pConn); free(data); return pConn; diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index 59d627c562..827942b0e2 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -244,5 +244,5 @@ char *tsError[] = {"success", "failed to lock resources", "table id/uid mismatch", "client query cache erased", // 119 - "too many authentication failed, try 10 minutes later", //120 + "too many authentication failed, try 10 minutes later", //120 }; diff --git a/src/rpc/src/ttcpclient.c b/src/rpc/src/ttcpclient.c index 3d39be92fe..297e916fbb 100644 --- a/src/rpc/src/ttcpclient.c +++ b/src/rpc/src/ttcpclient.c @@ -206,17 +206,20 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno)); + free(pTcp); return NULL; } if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); + free(pTcp); return NULL; } pTcp->pollFd = epoll_create(10); // size does not matter if (pTcp->pollFd < 0) { tError("%s failed to create TCP epoll", label); + free(pTcp); return NULL; } @@ -226,6 +229,7 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) { tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); + free(pTcp); return NULL; } diff --git a/src/rpc/src/ttcpserver.c b/src/rpc/src/ttcpserver.c index 663bfcdf8e..9d2c4e713d 100644 --- a/src/rpc/src/ttcpserver.c +++ b/src/rpc/src/ttcpserver.c @@ -389,6 +389,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); + free(pServerObj); return NULL; } memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); @@ -401,17 +402,23 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) { tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno)); + free(pServerObj->pThreadObj); + free(pServerObj); return NULL; } if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); + free(pServerObj->pThreadObj); + free(pServerObj); return NULL; } pThreadObj->pollFd = epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); + free(pServerObj->pThreadObj); + free(pServerObj); return NULL; } @@ -419,6 +426,8 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); + free(pServerObj->pThreadObj); + free(pServerObj); return NULL; } @@ -430,6 +439,8 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); + free(pServerObj->pThreadObj); + free(pServerObj); return NULL; } diff --git a/src/sdb/inc/sdbint.h b/src/sdb/inc/sdbint.h index c5b4f4e4ae..3327c1f731 100644 --- a/src/sdb/inc/sdbint.h +++ b/src/sdb/inc/sdbint.h @@ -127,7 +127,7 @@ typedef struct { } SMnodeStatus; typedef struct { - uint8_t dbId; + char dbId; char type; uint64_t version; short dataLen; diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c index 77b6f6d958..e2bde1eb8b 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/sdb/src/sdbEngine.c @@ -289,7 +289,7 @@ sdb_exit1: return -1; } -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, +void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, char keyType, char *directory, void *(*appTool)(char, void *, char *, int, int *)) { SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); if (pTable == NULL) return NULL; @@ -310,7 +310,7 @@ void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, pTable->appTool = appTool; sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); + if (sdbInitIndexFp[(int)keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[(int)keyType])(maxRows, sizeof(SRowMeta)); pthread_mutex_init(&pTable->mutex, NULL); diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index cade9b6d39..424b65a0dd 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -176,9 +176,9 @@ typedef struct _user_obj { char writeAuth : 1; char reserved[16]; char updateEnd[1]; + struct _user_obj *prev, *next; int32_t authAllowTime; int32_t authFailCount; - struct _user_obj *prev, *next; } SUserObj; typedef struct { @@ -432,6 +432,8 @@ bool mgmtAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode); void mgmtSetModuleInDnode(SDnodeObj *pDnode, int moduleType); int mgmtUnSetModuleInDnode(SDnodeObj *pDnode, int moduleType); +void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes); + extern int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); extern int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn); diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index d5ceac2a32..6dfc4709ed 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -261,7 +261,7 @@ typedef struct SQuery { TSKEY ekey; int64_t intervalTime; int64_t slidingTime; // sliding time for sliding window query - char intervalTimeUnit; // interval data type, used for daytime revise + char slidingTimeUnit; // interval data type, used for daytime revise int8_t precision; int16_t numOfOutputCols; int16_t interpoType; diff --git a/src/system/detail/inc/vnodeQueryImpl.h b/src/system/detail/inc/vnodeQueryImpl.h index c26778b328..71318ca797 100644 --- a/src/system/detail/inc/vnodeQueryImpl.h +++ b/src/system/detail/inc/vnodeQueryImpl.h @@ -85,12 +85,6 @@ typedef enum { QUERY_NO_DATA_TO_CHECK = 0x8u, } vnodeQueryStatus; -typedef struct SPointInterpoSupporter { - int32_t numOfCols; - char** pPrevPoint; - char** pNextPoint; -} SPointInterpoSupporter; - typedef struct SBlockInfo { TSKEY keyFirst; TSKEY keyLast; @@ -285,6 +279,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo); void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); +SWindowResult* getWindowRes(SWindowResInfo* pWindowResInfo, size_t index); #ifdef __cplusplus } diff --git a/src/system/detail/inc/vnodeRead.h b/src/system/detail/inc/vnodeRead.h index 0e5f40cd47..0b636db110 100644 --- a/src/system/detail/inc/vnodeRead.h +++ b/src/system/detail/inc/vnodeRead.h @@ -141,6 +141,12 @@ typedef struct SWindowResInfo { int64_t threshold; // threshold for return completed results. } SWindowResInfo; +typedef struct SPointInterpoSupporter { + int32_t numOfCols; + char** pPrevPoint; + char** pNextPoint; +} SPointInterpoSupporter; + typedef struct SQueryRuntimeEnv { SPositionInfo startPos; /* the start position, used for secondary/third iteration */ SPositionInfo endPos; /* the last access position in query, served as the start pos of reversed order query */ @@ -172,6 +178,10 @@ typedef struct SQueryRuntimeEnv { bool stableQuery; // is super table query or not SQueryDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + bool hasTimeWindow; + char** lastRowInBlock; + bool interpoSearch; + /* * Temporarily hold the in-memory cache block info during scan cache blocks * Here we do not use the cache block info from pMeterObj, simple because it may change anytime diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index a2a6ed8a7d..207443fb8e 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -567,6 +567,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMetric = mgmtGetMeter(pTagData); if (pMetric == NULL) { mError("table:%s, corresponding super table does not exist", pCreate->meterId); + free(pMeter); return TSDB_CODE_INVALID_TABLE; } diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index e4311bed3d..d99b729aa6 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1185,7 +1185,7 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { char * pStart, *pMsg; int msgLen; STaosRsp *pRsp; - + mgmtSaveQueryStreamList(cont, contLen, pConn); pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_HEARTBEAT_RSP, 128); @@ -1203,6 +1203,10 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { pConn->streamId = 0; pHBRsp->killConnection = pConn->killConnection; + mgmtGetDnodeOnlineNum(&pHBRsp->totalDnodes, &pHBRsp->onlineDnodes); + pHBRsp->totalDnodes = htonl(pHBRsp->totalDnodes); + pHBRsp->onlineDnodes = htonl(pHBRsp->onlineDnodes); + if (pConn->usePublicIp) { if (pSdbPublicIpList != NULL) { int size = pSdbPublicIpList->numOfIps * 4; @@ -1348,38 +1352,37 @@ _rsp: pRsp = (STaosRsp *)pMsg; pRsp->code = code; pMsg += sizeof(STaosRsp); - + pConnectRsp = (SConnectRsp *)pRsp->more; - if (NULL != pConn->pAcct) { - sprintf(pConnectRsp->acctId, "%x", pConn->pAcct->acctId); - } - strcpy(pConnectRsp->version, version); - pConnectRsp->writeAuth = pConn->writeAuth; - pConnectRsp->superAuth = pConn->superAuth; - pMsg += sizeof(SConnectRsp); - int size; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); - if (pConn->usePublicIp) { - memcpy(pMsg, pSdbPublicIpList, size); + if (code == 0) { + sprintf(pConnectRsp->acctId, "%x", pConn->pAcct->acctId); + strcpy(pConnectRsp->version, version); + pConnectRsp->writeAuth = pConn->writeAuth; + pConnectRsp->superAuth = pConn->superAuth; + pMsg += sizeof(SConnectRsp); + + int size; + if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { + size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); + if (pConn->usePublicIp) { + memcpy(pMsg, pSdbPublicIpList, size); + } else { + memcpy(pMsg, pSdbIpList, size); + } } else { - memcpy(pMsg, pSdbIpList, size); + SIpList tmpIpList; + tmpIpList.numOfIps = 0; + size = tmpIpList.numOfIps * 4 + sizeof(SIpList); + memcpy(pMsg, &tmpIpList, size); } - } else { - SIpList tmpIpList; - tmpIpList.numOfIps = 0; - size = tmpIpList.numOfIps * 4 + sizeof(SIpList); - memcpy(pMsg, &tmpIpList, size); - } - pMsg += size; - - // set the time resolution: millisecond or microsecond - *((uint32_t *)pMsg) = tsTimePrecision; - pMsg += sizeof(uint32_t); + pMsg += size; - if (code != 0) { + // set the time resolution: millisecond or microsecond + *((uint32_t *)pMsg) = tsTimePrecision; + pMsg += sizeof(uint32_t); + } else { pConnectRsp->writeAuth = 0; pConnectRsp->superAuth = 0; pConn->pAcct = NULL; diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index 347b54595e..b1caacd3a4 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -577,7 +577,7 @@ static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { const wchar_t* pattern = pRight; const wchar_t* str = pLeft; - int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + int32_t ret = WCSPatternMatch(pattern, str, twcslen(str), &pInfo); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } @@ -711,7 +711,6 @@ static int32_t mgmtFilterMeterByIndex(STabObj* pMetric, tQueryResultset* pRes, c // failed to build expression, no result, return immediately if (pExpr == NULL) { mError("metric:%s, no result returned, error in super table query expression:%s", pMetric->meterId, pCond); - tfree(pCond); return TSDB_CODE_OPS_NOT_SUPPORT; } else { // query according to the binary expression diff --git a/src/system/detail/src/mgmtUser.c b/src/system/detail/src/mgmtUser.c index 89b83e3553..b854a62630 100644 --- a/src/system/detail/src/mgmtUser.c +++ b/src/system/detail/src/mgmtUser.c @@ -184,6 +184,12 @@ int mgmtGetUserMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = TSDB_USER_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "account"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -230,6 +236,10 @@ int mgmtRetrieveUsers(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { *(int64_t *)pWrite = pUser->createdTime; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, pUser->acct); + cols++; + numOfRows++; } pShow->numOfReads += numOfRows; diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index 8ac2f21211..d1ccbe625c 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -413,7 +413,7 @@ void vnodeRemoveFile(int vnode, int fileId) { vnodeGetDnameFromLname(headName, dataName, lastName, dHeadName, dDataName, dLastName); int fd = open(headName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd > 0) { + if (fd >= 0) { vnodeGetHeadFileHeaderInfo(fd, &headInfo); atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), -headInfo.totalStorage); close(fd); diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 44fa7062db..4adb9ed6c4 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -35,6 +35,8 @@ #include "vnodeQueryImpl.h" #include "vnodeStatus.h" +#include + enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, @@ -80,6 +82,17 @@ static int32_t getGroupResultId(int32_t groupIndex) { return base + (groupIndex * 10000); } +static bool needsBoundaryTS(SQuery *pQuery) { + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { + return true; + } + } + + return false; +} + static FORCE_INLINE bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } // check the offset value integrity @@ -579,9 +592,9 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { return false; } -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t StartQueryTimestamp, void *inputData, +static void setExecParams(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int64_t StartQueryTimestamp, void *inputData, char *primaryColumnData, int32_t size, int32_t functionId, SField *pField, bool hasNull, - int32_t blockStatus, void *param, int32_t scanFlag); + void *param); void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo); @@ -1518,15 +1531,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.ekey = w.skey + pQuery->intervalTime - 1; } - /* - * query border check, skey should not be bounded by the query time range, since the value skey will - * be used as the time window index value. So we only change ekey of time window accordingly. - */ - if (w.ekey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) { - w.ekey = pQuery->ekey; - } - - assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + assert(ts >= w.skey && ts <= w.ekey/* && w.skey != 0*/); return w; } @@ -1643,7 +1648,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); } else { // set the current index to be the last unclosed window int32_t i = 0; - int64_t skey = 0; + int64_t skey = INT64_MIN; for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; @@ -1651,8 +1656,12 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + /* + * when the ekey equals to lastKey of current block, do NOT close it, since the interpolation may + * be involved. + */ + if ((pResult->window.ekey < lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->window.skey > lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { skey = pResult->window.skey; @@ -1661,7 +1670,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, } // all windows are closed, set the last one to be the skey - if (skey == 0) { + if (skey == INT64_MIN) { assert(i == pWindowResInfo->size); pWindowResInfo->curIndex = pWindowResInfo->size - 1; } else { @@ -1679,7 +1688,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, dTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n); } - assert(pWindowResInfo->prevSKey != 0); + assert(pWindowResInfo->prevSKey != INT64_MIN); } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SBlockInfo *pBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, @@ -1742,7 +1751,8 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat pCtx[k].size = forwardStep; pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1); - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 + || ((functionId >= TSDB_FUNC_RATE) && (functionId <= TSDB_FUNC_AVG_IRATE))) { pCtx[k].ptsList = (TSKEY *)((char*)pRuntimeEnv->primaryColBuffer->data + pCtx[k].startOffset * TSDB_KEYSIZE); } @@ -1829,6 +1839,31 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow } } +static void handleInBlockEkeyInterpolation(SQueryRuntimeEnv* pRuntimeEnv, int32_t endPos, + const TSKEY* primaryKeyCol, STimeWindow* win, SQLFunctionCtx* pCtx) { + // this query time window ended in the current data block + SQuery* pQuery = pRuntimeEnv->pQuery; + TSKEY lastKey = primaryKeyCol[endPos]; + + TSKEY e = win->skey + pQuery->intervalTime; + TSKEY next = primaryKeyCol[endPos + 1]; + + // the next key is beyond the query time range + if ((next > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || (next > pQuery->skey && !QUERY_IS_ASC_QUERY(pQuery))) { + pCtx->next.key = -1; + return; + } + + pCtx->next.key = e; + char *d = pCtx->aInputElemBuf + pCtx->inputBytes * endPos; + + SPoint point1 = (SPoint){.key = lastKey, .val = d}; + SPoint point2 = (SPoint){.key = next, .val = (d + pCtx->inputBytes)}; + SPoint point = (SPoint){.key = pCtx->next.key, .val = &pCtx->next.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); +} + static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { TSKEY ekey = -1; if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -1846,6 +1881,257 @@ static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { return ekey; } +static void interpolateEndKeyValue(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo* pBlockInfo, STimeWindow* win, + int32_t endPos, SQLFunctionCtx* pCtx, int32_t index) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + TSKEY *primaryKeyCol = (TSKEY *)pRuntimeEnv->primaryColBuffer->data; + + // if current query window beyonds the whole query window, do not employ the interpolation + if ((win->ekey >= pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (win->ekey >= pQuery->skey && !QUERY_IS_ASC_QUERY(pQuery))) { + pCtx->next.key = -1; + return; + } + + if (!QUERY_IS_ASC_QUERY(pQuery) && win->skey >= pBlockInfo->keyLast) { + pCtx->next.key = -1; + return; + } + + if (QUERY_IS_ASC_QUERY(pQuery)) { + + /* + * the time window closed before current data block, use the interpolation to generate + * the final result part, endPos equals to -1 means that this time window ends before current data block. + */ + if (win->ekey < pBlockInfo->keyFirst) { + assert(endPos == -1); + + TSKEY prev = *(int64_t*) pRuntimeEnv->lastRowInBlock[0]; + TSKEY next = pBlockInfo->keyFirst; + + pCtx->next.key = win->skey + pQuery->intervalTime; + + char *d = pCtx->aInputElemBuf; + + SPoint point1 = (SPoint){.key = prev, .val = pRuntimeEnv->lastRowInBlock[index]}; + SPoint point2 = (SPoint){.key = next, .val = d}; + SPoint point = (SPoint){.key = pCtx->next.key, .val = &pCtx->next.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); + } else if (win->ekey < pBlockInfo->keyLast) { + handleInBlockEkeyInterpolation(pRuntimeEnv, endPos, primaryKeyCol, win, pCtx); + } else { + //do nothing now, the interpolation will be handled before processing the next data block + assert(win->ekey >= pBlockInfo->keyLast); + pCtx->next.key = -1; + } + } else { // desc order query + + //the time window closed before current data block, use the interpolation to generate the final result part. + if (win->ekey >= pBlockInfo->keyLast) { + TSKEY prev = pBlockInfo->keyLast; + TSKEY next = *(TSKEY*) pRuntimeEnv->lastRowInBlock[0]; + + pCtx->next.key = win->skey + pQuery->intervalTime; + + char *d = pCtx->aInputElemBuf + (pBlockInfo->size - 1) * pCtx->inputBytes; + + SPoint point1 = (SPoint){.key = prev, .val = d}; + SPoint point2 = (SPoint){.key = next, .val = pRuntimeEnv->lastRowInBlock[index]}; + SPoint point = (SPoint){.key = pCtx->next.key, .val = &pCtx->next.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); + } else if (win->ekey < pBlockInfo->keyLast) { + handleInBlockEkeyInterpolation(pRuntimeEnv, endPos, primaryKeyCol, win, pCtx); + } else { + pCtx->next.key = -1; + } + } + +} + +static void handleInBlockSkeyInterpolation (SQueryRuntimeEnv* pRuntimeEnv, int32_t startPos, + const TSKEY* primaryKeyCol, STimeWindow* win, SQLFunctionCtx* pCtx) { + assert(startPos > 0); + + SQuery* pQuery = pRuntimeEnv->pQuery; + + TSKEY prev = primaryKeyCol[startPos - 1]; + TSKEY next = primaryKeyCol[startPos]; + + if (!QUERY_IS_ASC_QUERY(pQuery) && prev < pQuery->ekey) { + pCtx->prev.key = -1; + return; + } + + pCtx->prev.key = win->skey; + char *d = pCtx->aInputElemBuf + pCtx->inputBytes * (startPos - 1); + + SPoint point1 = (SPoint){.key = prev, .val = d}; + SPoint point2 = (SPoint){.key = next, .val = (d + pCtx->inputBytes)}; + SPoint point = (SPoint){.key = pCtx->prev.key, .val = &pCtx->prev.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); +} + +static void interpolateStartKeyValue(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo* pBlockInfo, SWindowResInfo* pWindowResInfo, + STimeWindow* win, int32_t startPos, SQLFunctionCtx* pCtx, int32_t index) { + SQuery* pQuery = pRuntimeEnv->pQuery; + TSKEY *primaryKeyCol = (TSKEY *)pRuntimeEnv->primaryColBuffer->data; + TSKEY skey = primaryKeyCol[startPos]; + + /* + * no need the start time interpolation + * 1. current window is the first window in either ascending or descending order output + * 2. time window start exactly from a timestamp with data + */ + if (skey == win->skey || win->skey < pWindowResInfo->startTime || + (win->skey <= pQuery->skey && QUERY_IS_ASC_QUERY(pQuery)) || + (win->skey <= pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + pCtx->prev.key = -1; + return; + } + + if (QUERY_IS_ASC_QUERY(pQuery)) { + // the queried time window and time window of data block must be intersect +// assert(win->ekey >= pBlockInfo->keyFirst && win->skey <= pBlockInfo->keyLast); + + /* + * this win should not be the first time window that starts from a less timestamp than + * the skey of current data block + */ + if (win->skey < pBlockInfo->keyFirst) { + TSKEY prev = *(TSKEY*) pRuntimeEnv->lastRowInBlock[0]; + TSKEY next = pBlockInfo->keyFirst; + + pCtx->prev.key = win->skey; + char *d = pCtx->aInputElemBuf; + + SPoint point1 = (SPoint){.key = prev, .val = pRuntimeEnv->lastRowInBlock[index]}; + SPoint point2 = (SPoint){.key = next, .val = d}; + SPoint point = (SPoint){.key = pCtx->prev.key, .val = &pCtx->prev.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); + } else { + handleInBlockSkeyInterpolation(pRuntimeEnv, startPos, primaryKeyCol, win, pCtx); + } + } else { // desc order + if (win->skey > pBlockInfo->keyLast) { + //this pBlockInfo located before current time window + TSKEY prev = pBlockInfo->keyLast; + TSKEY next = *(TSKEY*) pRuntimeEnv->lastRowInBlock[0]; + + pCtx->prev.key = win->skey; + char *d = pCtx->aInputElemBuf + (pBlockInfo->size - 1) * pCtx->inputBytes; + + SPoint point1 = (SPoint){.key = prev, .val = d}; + SPoint point2 = (SPoint){.key = next, .val = pRuntimeEnv->lastRowInBlock[index]}; + SPoint point = (SPoint){.key = pCtx->prev.key, .val = &pCtx->prev.data}; + + taosDoLinearInterpolationD(pCtx->inputType, &point1, &point2, &point); + } else { + // the queried time window and time window of data block must be intersect + assert(win->ekey >= pBlockInfo->keyFirst && win->skey <= pBlockInfo->keyLast); + + if (win->skey >= pBlockInfo->keyFirst) { + // the pBlockInfo is intersected with query time window + handleInBlockSkeyInterpolation(pRuntimeEnv, startPos, primaryKeyCol, win, pCtx); + } else { + assert(win->skey < pBlockInfo->keyFirst && win->ekey >= pBlockInfo->keyFirst); + pCtx->prev.key = -1; + } + } + } +} + +static void doSetInterpolationDataForTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, + SBlockInfo* pBlockInfo, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (!pRuntimeEnv->interpoSearch) { + return; + } + + int32_t s = startPos; + int32_t e = forwardStep * step + startPos - step; + + if (!QUERY_IS_ASC_QUERY(pQuery)) { + SWAP(s, e, int32_t); + } + + // interpolate for skey value + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + if ((pQuery->pSelectExpr[i].pBase.functionId < TSDB_FUNC_RATE) + || (pQuery->pSelectExpr[i].pBase.functionId > TSDB_FUNC_AVG_IRATE)) { + continue; + } + + SColIndexEx *pCol = &pQuery->pSelectExpr[i].pBase.colInfo; + interpolateStartKeyValue(pRuntimeEnv, pBlockInfo, pWindowResInfo, win, s, &pRuntimeEnv->pCtx[i], pCol->colIdxInBuf); + } + + // interpolate for ekey value + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + if ((pQuery->pSelectExpr[i].pBase.functionId < TSDB_FUNC_RATE) + || (pQuery->pSelectExpr[i].pBase.functionId > TSDB_FUNC_AVG_IRATE)) { + continue; + } + + SColIndexEx *pCol = &pQuery->pSelectExpr[i].pBase.colInfo; + interpolateEndKeyValue(pRuntimeEnv, pBlockInfo, win, e, &pRuntimeEnv->pCtx[i], pCol->colIdxInBuf); + } +} + +static void doInterpolatePrevTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo, SBlockInfo* pBlockInfo, + TSKEY ts, int32_t offset, STimeWindow* win) { + // get current not closed time window + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t slot = pWindowResInfo->curIndex; + if (slot == -1 || !pRuntimeEnv->interpoSearch) { + return; + } + + while (slot < pWindowResInfo->size) { + STimeWindow w = getWindowResult(pWindowResInfo, slot)->window; + if (w.skey == win->skey) { + assert(w.ekey == win->ekey); + break; + } + + // do not check for the closed time window + SWindowResult* pWindowRes = getWindowRes(pWindowResInfo, slot); + if (pWindowRes->status.closed) { + slot += 1; + continue; + } + + // if current active window locates before current data block, do interpolate the result and close it + assert((w.skey < win->skey && w.ekey < ts && QUERY_IS_ASC_QUERY(pQuery)) || + (w.skey > win->skey && w.skey > ts && !QUERY_IS_ASC_QUERY(pQuery))); + + int32_t forwardStep = 0; + doSetInterpolationDataForTimeWindow(pRuntimeEnv, pWindowResInfo, pBlockInfo, &w, offset, forwardStep); + + // set correct output buffer for interplate result. todo handle error + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pMeterObj->sid, &w) != TSDB_CODE_SUCCESS) { + continue; + } + + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, slot); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &w, pQuery->pos, forwardStep); + + closeTimeWindow(pWindowResInfo, slot); + + // try next time window + slot += 1; + } +} + /** * * @param pRuntimeEnv @@ -1872,7 +2158,6 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SField dummyField = {0}; - bool hasNull = hasNullVal(pQuery, k, pBlockInfo, pFields, isDiskFileBlock); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, forwardStep); @@ -1890,29 +2175,33 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t } } - setExecParams(pQuery, &pCtx[k], pQuery->skey, dataBlock, (char *)primaryKeyCol, forwardStep, functionId, tpField, - hasNull, pRuntimeEnv->blockStatus, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pRuntimeEnv, &pCtx[k], pQuery->skey, dataBlock, (char *)primaryKeyCol, forwardStep, functionId, tpField, + hasNull, &sasArray[k]); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (isIntervalQuery(pQuery)) { int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); TSKEY ts = primaryKeyCol[offset]; - + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + doInterpolatePrevTimeWindow(pRuntimeEnv, pWindowResInfo, pBlockInfo, ts, offset, &win); + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pMeterObj->sid, &win) != TSDB_CODE_SUCCESS) { return 0; } TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true); - + + doSetInterpolationDataForTimeWindow(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, offset, forwardStep); + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep); - int32_t index = pWindowResInfo->curIndex; + int32_t index = pWindowResInfo->curIndex; + STimeWindow nextWin = win; - while (1) { int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pBlockInfo, primaryKeyCol, searchFn); @@ -1928,7 +2217,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); - + + doSetInterpolationDataForTimeWindow(pRuntimeEnv, pWindowResInfo, pBlockInfo, &nextWin, startPos, forwardStep); + pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep); } @@ -1942,6 +2233,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t */ for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + pCtx[k].next.key = -1; + pCtx[k].prev.key = -1; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); } @@ -1956,6 +2250,14 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t if (!isIntervalQuery(pQuery)) { num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; } + + // save the last row in current data block + for(int32_t i = 0; i < pQuery->numOfCols; ++i) { + SColumnInfo* pColInfo = &pQuery->colList[i].data; + int32_t s = (QUERY_IS_ASC_QUERY(pQuery))? pColInfo->bytes * (pBlockInfo->size - 1) : 0; + + memcpy(pRuntimeEnv->lastRowInBlock[i], pRuntimeEnv->colDataBuffer[i]->data + s, pColInfo->bytes); + } tfree(sasArray); return (int32_t)num; @@ -2166,6 +2468,31 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { } } +SWindowResult* getWindowRes(SWindowResInfo* pWindowResInfo, size_t index) { + assert(index < pWindowResInfo->size); + return &pWindowResInfo->pResult[index]; +} + +/* + * remove the results that are not the FIRST time window that spreads beyond the + * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time + */ +void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { + assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); + + int32_t i = 0; + while(i < pWindowResInfo->size && + ((pWindowResInfo->pResult[i].window.ekey < lastKey && order == QUERY_ASC_FORWARD_STEP) || + (pWindowResInfo->pResult[i].window.skey > lastKey && order == QUERY_DESC_FORWARD_STEP))) { + ++i; + } + +// assert(i < pWindowResInfo->size); + if (i < pWindowResInfo->size) { + pWindowResInfo->size = (i + 1); + } +} + static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { if (isNull(pData, type)) { // ignore the null value return -1; @@ -2311,10 +2638,9 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * bool hasNull = hasNullVal(pQuery, k, pBlockInfo, pFields, isDiskFileBlock); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, *forwardStep); - TSKEY ts = pQuery->skey; // QUERY_IS_ASC_QUERY(pQuery) ? pRuntimeEnv->intervalWindow.skey : - // pRuntimeEnv->intervalWindow.ekey; - setExecParams(pQuery, &pCtx[k], ts, dataBlock, (char *)primaryKeyCol, (*forwardStep), functionId, pFields, hasNull, - pRuntimeEnv->blockStatus, &sasArray[k], pRuntimeEnv->scanFlag); + TSKEY ts = pQuery->skey; + setExecParams(pRuntimeEnv, &pCtx[k], ts, dataBlock, (char *)primaryKeyCol, (*forwardStep), functionId, pFields, hasNull, + &sasArray[k]); } // set the input column data @@ -2340,7 +2666,9 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * int32_t j = 0; TSKEY lastKey = -1; - + int32_t lastIndex = -1; + //bool firstAccessedPoint = true; + for (j = 0; j < (*forwardStep); ++j) { int32_t offset = GET_COL_DATA_POS(pQuery, j, step); @@ -2362,8 +2690,31 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * // interval window query if (isIntervalQuery(pQuery)) { // decide the time window according to the primary timestamp - int64_t ts = primaryKeyCol[offset]; + TSKEY ts = primaryKeyCol[offset]; + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); +// if (firstAccessedPoint) { +// doInterpolatePrevTimeWindow(pRuntimeEnv, pWindowResInfo, pBlockInfo, ts, offset, &win); +// firstAccessedPoint = false; +// } else { +// int32_t index = pWindowResInfo->curIndex; +// STimeWindow w = getWindowResult(pWindowResInfo, index)->window; +// +// if (w.skey == win.skey) { // do nothing +// assert(w.ekey == win.ekey); +// } else { +// assert((w.skey < win.skey && w.ekey < ts && QUERY_IS_ASC_QUERY(pQuery)) || +// (w.skey > win.skey && w.skey > ts && !QUERY_IS_ASC_QUERY(pQuery))); +// +// // set the endkey interpolation for the previous +// for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { +// SColIndexEx *pCol = &pQuery->pSelectExpr[i].pBase.colInfo; +// +// interpolateEndKeyValue(pRuntimeEnv, pBlockInfo, win, e, &pRuntimeEnv->pCtx[i], pCol->colIdxInBuf); +// } +// +// } +// } int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pMeterObj->sid, &win); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code @@ -2377,6 +2728,8 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); lastKey = ts; + lastIndex = j; + STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; int32_t sid = pRuntimeEnv->pMeterObj->sid; @@ -2421,6 +2774,9 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + pCtx[k].next.key = -1; + pCtx[k].prev.key = -1; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); } @@ -2434,7 +2790,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * break; } } - + /* * pointsOffset is the maximum available space in result buffer update the actual forward step for query that * requires checking buffer during loop @@ -2445,6 +2801,15 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * break; } } + + // save the last accessed row of current data block for interpolation + int32_t index = GET_COL_DATA_POS(pQuery, lastIndex, step); + for(int32_t i = 0; i < pQuery->numOfCols; ++i) { + SColumnInfo* pColInfo = &pQuery->colList[i].data; + int32_t s = pColInfo->bytes * index; + + memcpy(pRuntimeEnv->lastRowInBlock[i], pRuntimeEnv->colDataBuffer[i]->data + s, pColInfo->bytes); + } free(sasArray); @@ -2657,11 +3022,24 @@ int32_t getNextDataFileCompInfo(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeter return fileIndex; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t startQueryTimestamp, void *inputData, +static void getOneRowFromDataBlock(SQueryRuntimeEnv *pRuntimeEnv, char **dst, int32_t pos) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + int32_t bytes = pQuery->colList[i].data.bytes; + memcpy(dst[i], pRuntimeEnv->colDataBuffer[i]->data + pos * bytes, bytes); + } +} + +void setExecParams(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int64_t startQueryTimestamp, void *inputData, char *primaryColumnData, int32_t size, int32_t functionId, SField *pField, bool hasNull, - int32_t blockStatus, void *param, int32_t scanFlag) { + void *param) { + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); - + + int32_t scanFlag = pRuntimeEnv->scanFlag; + int32_t blockStatus = pRuntimeEnv->blockStatus; + pCtx->nStartQueryTimestamp = startQueryTimestamp; pCtx->scanFlag = scanFlag; @@ -2913,6 +3291,12 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pInterpoBuf); } + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + tfree(pRuntimeEnv->lastRowInBlock[i]); + } + + tfree(pRuntimeEnv->lastRowInBlock); + destroyDiskbasedResultBuf(pRuntimeEnv->pResultBuf); pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -3324,7 +3708,7 @@ void vnodeCheckIfDataExists(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeterObj, void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t pKey, int64_t keyFirst, int64_t keyLast, int64_t *actualSkey, int64_t *actualEkey, int64_t *skey, int64_t *ekey) { assert(pKey >= keyFirst && pKey <= keyLast); - *skey = taosGetIntervalStartTimestamp(pKey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision); + *skey = taosGetIntervalStartTimestamp(pKey, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); if (keyFirst > (INT64_MAX - pQuery->intervalTime)) { /* @@ -3355,13 +3739,62 @@ void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t pKey, int64_t ke } } -static void getOneRowFromDataBlock(SQueryRuntimeEnv *pRuntimeEnv, char **dst, int32_t pos) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - int32_t bytes = pQuery->colList[i].data.bytes; - memcpy(dst[i], pRuntimeEnv->colDataBuffer[i]->data + pos * bytes, bytes); +static bool loadPrevDataPoint(SQueryRuntimeEnv* pRuntimeEnv, char** result) { + SQuery* pQuery = pRuntimeEnv->pQuery; + SMeterObj* pMeterObj = pRuntimeEnv->pMeterObj; + + /* the qualified point is not the first point in data block */ + if (pQuery->pos > 0) { + int32_t prevPos = pQuery->pos - 1; + + /* save the point that is directly after the specified point */ + getOneRowFromDataBlock(pRuntimeEnv, result, prevPos); + } else { + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + + savePointPosition(&pRuntimeEnv->startPos, pQuery->fileId, pQuery->slot, pQuery->pos); + + // backwards movement would not set the pQuery->pos correct. We need to set it manually later. + moveToNextBlock(pRuntimeEnv, QUERY_DESC_FORWARD_STEP, searchFn, true); + + /* + * no previous data exists. + * reset the status and load the data block that contains the qualified point + */ + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { + dTrace("QInfo:%p no previous data block, start fileId:%d, slot:%d, pos:%d, qrange:%" PRId64 "-%" PRId64 + ", out of range", + GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, + pRuntimeEnv->startPos.pos, pQuery->skey, pQuery->ekey); + + // no result, return immediately + setQueryStatus(pQuery, QUERY_COMPLETED); + return false; + } else { // prev has been located + if (pQuery->fileId >= 0) { + pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, result, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); + + // restore to the start position + loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->startPos); + } else { + // moveToNextBlock make sure there is a available cache block, if exists + assert(vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, -1, true) == DISK_BLOCK_NO_NEED_TO_LOAD); + SCacheBlock* pBlock = &pRuntimeEnv->cacheBlock; + + pQuery->pos = pBlock->numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, result, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); + } + } } + + return true; } static bool getNeighborPoints(STableQuerySupportObj *pSupporter, SMeterObj *pMeterObj, @@ -3381,10 +3814,8 @@ static bool getNeighborPoints(STableQuerySupportObj *pSupporter, SMeterObj *pMet } else { assert(QUERY_IS_ASC_QUERY(pQuery)); } + assert(pPointInterpSupporter != NULL && pQuery->skey == pQuery->ekey); - - SCacheBlock *pBlock = NULL; - qTrace("QInfo:%p get next data point, fileId:%d, slot:%d, pos:%d", GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot, pQuery->pos); @@ -3413,55 +3844,9 @@ static bool getNeighborPoints(STableQuerySupportObj *pSupporter, SMeterObj *pMet } return true; } - - /* the qualified point is not the first point in data block */ - if (pQuery->pos > 0) { - int32_t prevPos = pQuery->pos - 1; - - /* save the point that is directly after the specified point */ - getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, prevPos); - } else { - __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; - - savePointPosition(&pRuntimeEnv->startPos, pQuery->fileId, pQuery->slot, pQuery->pos); - - // backwards movement would not set the pQuery->pos correct. We need to set it manually later. - moveToNextBlock(pRuntimeEnv, QUERY_DESC_FORWARD_STEP, searchFn, true); - - /* - * no previous data exists. - * reset the status and load the data block that contains the qualified point - */ - if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { - dTrace("QInfo:%p no previous data block, start fileId:%d, slot:%d, pos:%d, qrange:%" PRId64 "-%" PRId64 - ", out of range", - GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, - pRuntimeEnv->startPos.pos, pQuery->skey, pQuery->ekey); - - // no result, return immediately - setQueryStatus(pQuery, QUERY_COMPLETED); - return false; - } else { // prev has been located - if (pQuery->fileId >= 0) { - pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1; - getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); - - qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), - pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); - } else { - // moveToNextBlock make sure there is a available cache block, if exists - assert(vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, -1, true) == DISK_BLOCK_NO_NEED_TO_LOAD); - pBlock = &pRuntimeEnv->cacheBlock; - - pQuery->pos = pBlock->numOfPoints - 1; - getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); - - qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), - pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); - } - } - } - + + loadPrevDataPoint(pRuntimeEnv, pPointInterpSupporter->pPrevPoint); + pQuery->skey = *(TSKEY *)pPointInterpSupporter->pPrevPoint[0]; pQuery->ekey = *(TSKEY *)pPointInterpSupporter->pNextPoint[0]; pQuery->lastKey = pQuery->skey; @@ -3635,7 +4020,7 @@ bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySup if (key != NULL) { *key = nextKey; } - + return doGetQueryPos(nextKey, pSupporter, pPointInterpSupporter); } @@ -4223,63 +4608,6 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, STableQuerySupportObj * } } -// if (win.ekey <= blockInfo.keyLast) { -// pQuery->limit.offset -= 1; -// -// if (win.ekey == blockInfo.keyLast) { -// moveToNextBlock(pRuntimeEnv, step, searchFn, false); -// if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { -// break; -// } -// -// // next block does not included in time range, abort query -// blockInfo = getBlockInfo(pRuntimeEnv); -// if ((blockInfo.keyFirst > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || -// (blockInfo.keyLast < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { -// setQueryStatus(pQuery, QUERY_COMPLETED); -// break; -// } -// -// // set the window that start from the next data block -// win = getActiveTimeWindow(pWindowResInfo, blockInfo.keyFirst, pQuery); -// } else { -// // the time window is closed in current data block, load disk file block into memory to -// // check the next time window -// if (IS_DISK_DATA_BLOCK(pQuery)) { -// getTimestampInDiskBlock(pRuntimeEnv, 0); -// } -// -// STimeWindow nextWin = win; -// int32_t startPos = -// getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, &blockInfo, primaryKey, searchFn); -// -// if (startPos < 0) { // failed to find the qualified time window -// assert((nextWin.skey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || -// (nextWin.ekey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))); -// -// setQueryStatus(pQuery, QUERY_COMPLETED); -// break; -// } else { // set the abort info -// pQuery->pos = startPos; -// pQuery->lastKey = primaryKey[startPos]; -// win = nextWin; -// } -// } -// -// continue; -// } -// -// moveToNextBlock(pRuntimeEnv, step, searchFn, false); -// if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { -// break; -// } -// -// blockInfo = getBlockInfo(pRuntimeEnv); -// if ((blockInfo.keyFirst > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || -// (blockInfo.keyLast < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { -// setQueryStatus(pQuery, QUERY_COMPLETED); -// break; -// } } if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED) || pQuery->limit.offset > 0) { @@ -4320,7 +4648,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t len = t + 1 + TSDB_KEYSIZE; pCtx->param[index].pz = calloc(1, len); } else if (type == TSDB_DATA_TYPE_NCHAR) { - t = wcslen((const wchar_t *)data); + t = twcslen((const wchar_t *)data); len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE; pCtx->param[index].pz = calloc(1, len); @@ -4468,7 +4796,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI } void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { - if (isPointInterpoQuery(pQuery)) { + if (isPointInterpoQuery(pQuery) || needsBoundaryTS(pQuery)) { pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); @@ -4547,12 +4875,15 @@ static int32_t allocateRuntimeEnvBuf(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *p SQuery *pQuery = pRuntimeEnv->pQuery; // To make sure the start position of each buffer is aligned to 4bytes in 32-bit ARM system. + pRuntimeEnv->lastRowInBlock = calloc(pQuery->numOfCols, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfCols; ++i) { int32_t bytes = pQuery->colList[i].data.bytes; pRuntimeEnv->colDataBuffer[i] = calloc(1, sizeof(SData) + EXTRA_BYTES + pMeterObj->pointsPerFileBlock * bytes); if (pRuntimeEnv->colDataBuffer[i] == NULL) { goto _error_clean; } + + pRuntimeEnv->lastRowInBlock[i] = calloc(1, bytes); } // record the maximum column width among columns of this meter/metric @@ -4666,7 +4997,9 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pMeterObj = pMeterObj; - + pRuntimeEnv->hasTimeWindow = !notHasQueryTimeRange(pQuery); + pRuntimeEnv->interpoSearch = needsBoundaryTS(pQuery); + if ((code = allocateRuntimeEnvBuf(pRuntimeEnv, pMeterObj)) != TSDB_CODE_SUCCESS) { return code; } @@ -4721,9 +5054,9 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery /* query on single table */ pSupporter->numOfMeters = 1; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - - SPointInterpoSupporter interpInfo = {0}; - pointInterpSupporterInit(pQuery, &interpInfo); + + SPointInterpoSupporter interpoSupporter = {0}; + pointInterpSupporterInit(pQuery, &interpoSupporter); /* * in case of last_row query without query range, we set the query timestamp to @@ -4731,11 +5064,11 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery */ if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) { - if (!normalizeUnBoundLastRowQuery(pSupporter, &interpInfo)) { + if (!normalizeUnBoundLastRowQuery(pSupporter, &interpoSupporter)) { sem_post(&pQInfo->dataReady); pQInfo->over = 1; - pointInterpSupporterDestroy(&interpInfo); + pointInterpSupporterDestroy(&interpoSupporter); return TSDB_CODE_SUCCESS; } } else { // find the skey and ekey in case of sliding query @@ -4749,23 +5082,34 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery } int64_t skey = 0; - if ((normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &interpInfo, &skey) == false) || + if ((normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &interpoSupporter, &skey) == false) || (isFixedOutputQuery(pQuery) && !isTopBottomQuery(pQuery) && (pQuery->limit.offset > 0)) || (isTopBottomQuery(pQuery) && pQuery->limit.offset >= pQuery->pSelectExpr[1].pBase.arg[0].argValue.i64)) { sem_post(&pQInfo->dataReady); pQInfo->over = 1; - pointInterpSupporterDestroy(&interpInfo); + pointInterpSupporterDestroy(&interpoSupporter); return TSDB_CODE_SUCCESS; } + pQuery->skey = skey; if (!QUERY_IS_ASC_QUERY(pQuery)) { win.skey = minKey; win.ekey = skey; + pQuery->ekey = minKey; } else { win.skey = skey; win.ekey = pQuery->ekey; } + + // empty result + if (QUERY_IS_ASC_QUERY(pQuery) && win.skey > win.ekey) { + sem_post(&pQInfo->dataReady); + pQInfo->over = 1; + + pointInterpSupporterDestroy(&interpoSupporter); + return TSDB_CODE_SUCCESS; + } TSKEY skey1, ekey1; TSKEY windowSKey = 0, windowEKey = 0; @@ -4784,13 +5128,13 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery pQuery->over = QUERY_NOT_COMPLETED; } else { int64_t ekey = 0; - if ((normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &interpInfo, &ekey) == false) || + if ((normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &interpoSupporter, &ekey) == false) || (isFixedOutputQuery(pQuery) && !isTopBottomQuery(pQuery) && (pQuery->limit.offset > 0)) || (isTopBottomQuery(pQuery) && pQuery->limit.offset >= pQuery->pSelectExpr[1].pBase.arg[0].argValue.i64)) { sem_post(&pQInfo->dataReady); pQInfo->over = 1; - pointInterpSupporterDestroy(&interpInfo); + pointInterpSupporterDestroy(&interpoSupporter); return TSDB_CODE_SUCCESS; } } @@ -4800,14 +5144,14 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, STableQuery * here we set the value for before and after the specified time into the * parameter for interpolation query */ - pointInterpSupporterSetData(pQInfo, &interpInfo); - pointInterpSupporterDestroy(&interpInfo); + pointInterpSupporterSetData(pQInfo, &interpoSupporter); + pointInterpSupporterDestroy(&interpoSupporter); if (!forwardQueryStartPosIfNeeded(pQInfo, pSupporter, dataInDisk, dataInCache)) { return TSDB_CODE_SUCCESS; } - int64_t rs = taosGetIntervalStartTimestamp(pSupporter->rawSKey, pQuery->intervalTime, pQuery->intervalTimeUnit, + int64_t rs = taosGetIntervalStartTimestamp(pSupporter->rawSKey, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); allocMemForInterpo(pSupporter, pQuery, pMeterObj); @@ -4888,6 +5232,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pSupporter->rawEKey = pQuery->ekey; pSupporter->rawSKey = pQuery->skey; pQuery->lastKey = pQuery->skey; + pRuntimeEnv->interpoSearch = needsBoundaryTS(pQuery); // create runtime environment SColumnModel *pTagSchemaInfo = pSupporter->pSidSet->pColumnModel; @@ -4943,7 +5288,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { } TSKEY revisedStime = taosGetIntervalStartTimestamp(pSupporter->rawSKey, pQuery->intervalTime, - pQuery->intervalTimeUnit, pQuery->precision); + pQuery->slidingTimeUnit, pQuery->precision); taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, revisedStime, 0, 0); pRuntimeEnv->stableQuery = true; @@ -5389,6 +5734,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; } else if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { // check if window needs to be closed SBlockInfo blockInfo = getBlockInfo(pRuntimeEnv); @@ -5448,7 +5795,6 @@ void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, SM } // set the join tag for first column - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); @@ -5474,9 +5820,6 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes pCtx[i].hasNull = true; pCtx[i].nStartQueryTimestamp = timestamp; pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes); - // pCtx[i].aInputElemBuf = ((char *)inputSrc->data) + - // ((int32_t)pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage) + - // pCtx[i].outputBytes * inputIdx; // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -5837,6 +6180,7 @@ int32_t doMergeMetersResultsToGroupRes(STableQuerySupportObj *pSupporter, SQuery } else { // copy data to disk buffer if (buffer[0]->numOfElems == pQuery->pointsToRead) { if (flushFromResultBuf(pSupporter, pQuery, pRuntimeEnv) != TSDB_CODE_SUCCESS) { + tfree(pTree); return -1; } @@ -7171,7 +7515,6 @@ void setIntervalQueryRange(SMeterQueryInfo *pMeterQueryInfo, STableQuerySupportO doGetAlignedIntervalQueryRangeImpl(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &windowSKey, &windowEKey); pWindowResInfo->startTime = windowSKey; // windowSKey may be 0 in case of 1970 timestamp - // assert(pWindowResInfo->startTime > 0); if (pWindowResInfo->prevSKey == 0) { if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -7476,6 +7819,8 @@ void stableApplyFunctionsOnBlock(STableQuerySupportObj *pSupporter, SMeterDataIn updateWindowResNumOfRes(pRuntimeEnv, pMeterDataInfo); updatelastkey(pQuery, pMeterQueryInfo); + + doCheckQueryCompleted(pRuntimeEnv, pMeterQueryInfo->lastKey, pWindowResInfo); } // we need to split the refstatsult into different packages. @@ -7535,7 +7880,7 @@ bool vnodeHasRemainResults(void *handle) { // query has completed if (Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { TSKEY ekey = taosGetRevisedEndKey(pSupporter->rawEKey, pQuery->order.order, pQuery->intervalTime, - pQuery->intervalTimeUnit, pQuery->precision); + pQuery->slidingTimeUnit, pQuery->precision); int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pRuntimeEnv->pInterpoBuf[0]->data, remain, pQuery->intervalTime, ekey, pQuery->pointsToRead); return numOfTotal > 0; @@ -7646,7 +7991,7 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage numOfRows = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); TSKEY ekey = taosGetRevisedEndKey(pSupporter->rawEKey, pQuery->order.order, pQuery->intervalTime, - pQuery->intervalTimeUnit, pQuery->precision); + pQuery->slidingTimeUnit, pQuery->precision); int32_t numOfFinalRows = taosGetNumOfResultWithInterpo(&pRuntimeEnv->interpoInfo, (TSKEY *)pDataSrc[0]->data, numOfRows, pQuery->intervalTime, ekey, pQuery->pointsToRead); diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index d7b63119c6..7a92fa235b 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -269,7 +269,7 @@ static SQInfo *vnodeAllocateQInfoEx(SQueryMeterMsg *pQueryMsg, SSqlGroupbyExpr * pQuery->intervalTime = pQueryMsg->intervalTime; pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->interpoType = pQueryMsg->interpoType; - pQuery->intervalTimeUnit = pQueryMsg->intervalTimeUnit; + pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; pQInfo->query.pointsToRead = vnodeList[pMeterObj->vnode].cfg.rowsInFileBlock; @@ -496,6 +496,9 @@ void vnodeDecRefCount(void *param) { assert(vnodeIsQInfoValid(pQInfo)); int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); + if (ref < 0) { + return; // avoid two threads dec ref count + } assert(ref >= 0); dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); @@ -649,18 +652,28 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE } STableQuerySupportObj *pSupporter = (STableQuerySupportObj *)calloc(1, sizeof(STableQuerySupportObj)); + if (pSupporter == NULL) { + *code = TSDB_CODE_SERV_OUT_OF_MEMORY; + goto _error; + } pSupporter->numOfMeters = 1; - - pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false); - taosAddToHashTable(pSupporter->pMetersHashTable, (const char*) &pMetersObj[0]->sid, sizeof(pMeterObj[0].sid), - (char *)&pMetersObj[0], POINTER_BYTES); - pSupporter->pSidSet = NULL; pSupporter->subgroupIdx = -1; pSupporter->pMeterSidExtInfo = NULL; pQInfo->pTableQuerySupporter = pSupporter; + pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false); + if (pSupporter->pMetersHashTable == NULL) { + *code = TSDB_CODE_SERV_OUT_OF_MEMORY; + goto _error; + } + if (taosAddToHashTable(pSupporter->pMetersHashTable, (const char*) &pMetersObj[0]->sid, sizeof(pMeterObj[0].sid), + (char *)&pMetersObj[0], POINTER_BYTES) != 0) { + *code = TSDB_CODE_APP_ERROR; + goto _error; + } + STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index 5949b1636d..92e90ebe84 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -24,6 +24,8 @@ #include "vnodeUtil.h" #include "vnodeStatus.h" +#include + int tsMaxVnode = -1; int tsOpenVnodes = 0; SVnodeObj *vnodeList = NULL; diff --git a/src/system/lite/src/mgmtDnodeInt.spec.c b/src/system/lite/src/mgmtDnodeInt.spec.c index 734fa630c5..cc3351768f 100644 --- a/src/system/lite/src/mgmtDnodeInt.spec.c +++ b/src/system/lite/src/mgmtDnodeInt.spec.c @@ -67,6 +67,7 @@ int taosSendMsgToDnode(SDnodeObj *pObj, char *msg, int msgLen) { * Lite version has no message header, so minus one */ SSchedMsg schedMsg; + schedMsg.tfp = NULL; schedMsg.fp = vnodeProcessMsgFromMgmtSpec; schedMsg.msg = msg - 1; schedMsg.ahandle = NULL; diff --git a/src/system/lite/src/mgmtShell.spec.c b/src/system/lite/src/mgmtShell.spec.c index a1d8e6a34a..a95ecd2ee2 100644 --- a/src/system/lite/src/mgmtShell.spec.c +++ b/src/system/lite/src/mgmtShell.spec.c @@ -45,4 +45,9 @@ int mgmtProcessDropAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { int mgmtProcessCreateAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes) { + *totalDnodes = 1; + *onlineDnodes = 1; } \ No newline at end of file diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index d8f74f46f4..79304d976b 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () ENDIF () -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index c042c0181f..54c6ea039c 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -587,6 +587,8 @@ void *taosAddDataIntoCache(void *handle, char *key, char *pData, int dataSize, i "size:%" PRId64 " bytes, collision:%d", pNode->key, pNode, HASH_INDEX(pNode->hashVal, pObj->capacity), pNode->addTime, pNode->time, pObj->size, pObj->totalSize, pObj->statistics.numOfCollision); + } else { + pError("key:%s failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pObj, pOldNode, key, keyLen, pData, dataSize, keepTime * 1000L); diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 0f0c7bed34..3037d1bba7 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -769,7 +769,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char delta_of_delta = 0; } else { if (is_bigendian()) { - memcpy(&dd1 + LONG_BYTES - nbytes, input + ipos, nbytes); + memcpy(((char *)(&dd1)) + LONG_BYTES - nbytes, input + ipos, nbytes); } else { memcpy(&dd1, input + ipos, nbytes); } @@ -794,7 +794,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char delta_of_delta = 0; } else { if (is_bigendian()) { - memcpy(&dd2 + LONG_BYTES - nbytes, input + ipos, nbytes); + memcpy(((char *)(&dd2)) + LONG_BYTES - nbytes, input + ipos, nbytes); } else { memcpy(&dd2, input + ipos, nbytes); } diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index 8ce090d335..0d9d15b6af 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -41,11 +41,13 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { char *tmpDir = "/tmp/"; #endif + int64_t ts = taosGetTimestampUs(); + strcpy(tmpPath, tmpDir); strcat(tmpPath, tdengineTmpFileNamePrefix); strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%llu-%u"); - snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1)); + strcat(tmpPath, "-%d-%llu-%u-%llu"); + snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, getpid(), taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1), ts); } /* @@ -80,7 +82,7 @@ tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnMo return pMemBuffer; } -void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { +void* destroyExtMemBuffer(tExtMemBuffer *pMemBuffer) { if (pMemBuffer == NULL) { return NULL; } @@ -914,6 +916,7 @@ void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int3 char buf[4096] = {0}; taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); printf("%s\t", buf); + break; } case TSDB_DATA_TYPE_BINARY: { printBinaryData(val, pModel->pFields[j].field.bytes); @@ -965,6 +968,7 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in char buf[128] = {0}; taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); printf("%s\t", buf); + break; } case TSDB_DATA_TYPE_BINARY: { printBinaryDataEx(val, pModel->pFields[j].field.bytes, ¶m[j]); diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 050480ba19..915c1dc2fa 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -156,11 +156,11 @@ char tsSocketType[4] = "udp"; // time precision, millisecond by default int tsTimePrecision = TSDB_TIME_PRECISION_MILLI; -// 10 ms for sliding time, the value will changed in case of time precision changed -int tsMinSlidingTime = 10; +// 1 us for sliding time, the value will changed in case of time precision changed +int tsMinSlidingTime = 1; -// 10 ms for interval time range, changed accordingly -int tsMinIntervalTime = 10; +// 1 us for interval time range, changed accordingly +int tsMinIntervalTime = 1; // 20sec, the maximum value of stream computing delay, changed accordingly int tsMaxStreamComputDelay = 20000; @@ -632,10 +632,10 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 10, 1000000, 0, TSDB_CFG_UTYPE_MS); + 1, 1000000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "minIntervalTime", &tsMinIntervalTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 10, 1000000, 0, TSDB_CFG_UTYPE_MS); + 1, 1000000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "maxStreamCompDelay", &tsMaxStreamComputDelay, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000000, 0, TSDB_CFG_UTYPE_MS); @@ -690,11 +690,11 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "maxSQLLength", &tsMaxSQLStringLen, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_BYTE); - + tsInitConfigOption(cfg++, "maxAuthRetryTime", &tsMaxAuthRetry, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, 1, 10, 0, TSDB_CFG_UTYPE_BYTE); - + tsInitConfigOption(cfg++, "maxNumOfOrderedRes", &tsMaxNumOfOrderedResults, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_NONE); diff --git a/src/util/src/tinterpolation.c b/src/util/src/tinterpolation.c index cb7c8854ce..d86f3e44c5 100644 --- a/src/util/src/tinterpolation.c +++ b/src/util/src/tinterpolation.c @@ -22,12 +22,12 @@ #define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSQL_SO_ASC) -int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision) { +int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char slidingTimeUnit, int16_t precision) { if (timeRange == 0) { return startTime; } - if (intervalTimeUnit == 'a' || intervalTimeUnit == 'm' || intervalTimeUnit == 's' || intervalTimeUnit == 'h') { + if (slidingTimeUnit == 'a' || slidingTimeUnit == 'm' || slidingTimeUnit == 's' || slidingTimeUnit == 'h' || slidingTimeUnit == 'u') { return (startTime / timeRange) * timeRange; } else { /* @@ -95,11 +95,11 @@ void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawD pInterpoInfo->numOfRawDataInRows = numOfRawDataInRows; } -TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision) { +TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t slidingTimeUnit, int8_t precision) { if (order == TSQL_SO_ASC) { return ekey; } else { - return taosGetIntervalStartTimestamp(ekey, timeInterval, intervalTimeUnit, precision); + return taosGetIntervalStartTimestamp(ekey, timeInterval, slidingTimeUnit, precision); } } @@ -191,6 +191,49 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi return 0; } +int taosDoLinearInterpolationD(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { + switch (type) { + case TSDB_DATA_TYPE_INT: { + *(double*) point->val = doLinearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, point1->key, + point2->key, point->key); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(double*)point->val = + doLinearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, point1->key, point2->key, point->key); + break; + }; + case TSDB_DATA_TYPE_DOUBLE: { + *(double*)point->val = + doLinearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, point1->key, point2->key, point->key); + break; + }; + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: { + *(double*)point->val = doLinearInterpolationImpl(*(int64_t*)point1->val, *(int64_t*)point2->val, point1->key, + point2->key, point->key); + break; + }; + case TSDB_DATA_TYPE_SMALLINT: { + *(double*)point->val = doLinearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, point1->key, + point2->key, point->key); + break; + }; + case TSDB_DATA_TYPE_TINYINT: { + *(double*)point->val = + doLinearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, point1->key, point2->key, point->key); + break; + }; + default: { + // TODO: Deal with interpolation with bool and strings and timestamp + return -1; + } + } + + return 0; +} + + static char* getPos(char* data, int32_t bytes, int32_t index) { return data + index * bytes; } static void setTagsValueInInterpolation(tFilePage** data, char** pTags, SColumnModel* pModel, int32_t order, diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 7a133590d2..1a1b3a644d 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -77,7 +77,7 @@ void taosUnLockNote(int fd, taosNoteInfo * pNote) void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN]; + char name[NOTE_FILE_NAME_LEN + 16]; taosNoteInfo * pNote = (taosNoteInfo *)param; pNote->taosNoteFlag ^= 1; @@ -170,7 +170,7 @@ void taosGetNoteName(char *fn, taosNoteInfo * pNote) int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote) { - char name[NOTE_FILE_NAME_LEN] = "\0"; + char name[NOTE_FILE_NAME_LEN + 16] = "\0"; struct stat notestat0, notestat1; int size; diff --git a/src/util/src/tpercentile.c b/src/util/src/tpercentile.c index b3c09033b4..e013a2a33a 100644 --- a/src/util/src/tpercentile.c +++ b/src/util/src/tpercentile.c @@ -32,7 +32,7 @@ tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, i pBuffer = pSeg->pBuffer[j]; } else { if (pSeg->pBuffer && pSeg->pBuffer[j]) { - pSeg->pBuffer[j] = destoryExtMemBuffer(pSeg->pBuffer[j]); + pSeg->pBuffer[j] = destroyExtMemBuffer(pSeg->pBuffer[j]); } } } @@ -338,7 +338,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { if (pSeg->pBuffer[j] != NULL) { - pSeg->pBuffer[j] = destoryExtMemBuffer(pSeg->pBuffer[j]); + pSeg->pBuffer[j] = destroyExtMemBuffer(pSeg->pBuffer[j]); } } tfree(pSeg->pBuffer); @@ -588,7 +588,7 @@ void releaseBucket(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx) { return; } - pSeg->pBuffer[slotIdx] = destoryExtMemBuffer(pSeg->pBuffer[slotIdx]); + pSeg->pBuffer[slotIdx] = destroyExtMemBuffer(pSeg->pBuffer[slotIdx]); } //////////////////////////////////////////////////////////////////////////////////////////// @@ -853,7 +853,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) tMemBucketSegment *pSeg = &pMemBucket->pSegs[tt]; for (int32_t ttx = 0; ttx < pSeg->numOfSlots; ++ttx) { if (pSeg->pBuffer && pSeg->pBuffer[ttx]) { - pSeg->pBuffer[ttx] = destoryExtMemBuffer(pSeg->pBuffer[ttx]); + pSeg->pBuffer[ttx] = destroyExtMemBuffer(pSeg->pBuffer[ttx]); } } } diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 88231bd29c..9e0f92475e 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -392,6 +392,8 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu break; case 'a': break; + case 'u': + return 0; default: { ; return -1; diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index 7cbb4552b4..7cef1f7094 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -510,7 +510,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { /* here is the 1a/2s/3m/9y */ if ((z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' || z[i] == 'y' || z[i] == 'w' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || - z[i] == 'Y' || z[i] == 'W') && + z[i] == 'Y' || z[i] == 'W' || z[i] == 'u' || z[i] == 'U') && (isIdChar[(uint8_t)z[i + 1]] == 0)) { *tokenType = TK_VARIABLE; i += 1; diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index ae994cb77b..f5b61481cb 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -197,7 +197,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; - taosUcs4ToMbs(pVar->wpz, (wcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); + taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); int32_t len = strlen(dst); dst[len] = '\''; dst[len + 1] = 0; @@ -430,7 +430,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } pVariant->wpz = pWStr; - *pDestSize = wcslen(pVariant->wpz); + *pDestSize = twcslen(pVariant->wpz); // shrink the allocate memory, no need to check here. char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 21d147d947..9236c2860d 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -27,6 +27,23 @@ #include "tlog.h" #include "taoserror.h" +size_t twcslen(const wchar_t *wcs) { + int *wstr = (int *)wcs; + if (NULL == wstr) { + return 0; + } + + size_t n = 0; + while (1) { + if (0 == *wstr++) { + break; + } + n++; + } + + return n; +} + int32_t strdequote(char *z) { if (z == NULL) { return 0; diff --git a/src/util/src/version.c b/src/util/src/version.c index 80a62f453a..b8b9cbb734 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,7 +1,7 @@ -char version[64] = "1.6.6.1"; -char compatible_version[64] = "1.6.0.0"; -char gitinfo[128] = "feea817446e25ff1ef77cefaeeda08f45564d6bc"; -char gitinfoOfInternal[128] = "5c736133836e7a9f757216539986ddf746439a11"; -char buildinfo[512] = "Built by root at 2020-03-13 16:27"; +char version[64] = "2.0.2.3"; +char compatible_version[64] = "2.0.0.0"; +char gitinfo[128] = "615410abe06a3a5354385760ce9f46fb954a67c5"; +char gitinfoOfInternal[128] = "be258cbbe8a9bc3a3da785d451479e46e9c8d03e"; +char buildinfo[512] = "Built by root at 2020-05-27 14:12"; -void libtaos_1_6_6_1_Linux_x64() {}; +void libtaos_2_0_2_3_Linux_x64() {}; diff --git a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java b/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java index a7a1f7fc68..7bdc9411f6 100644 --- a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java +++ b/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java @@ -108,7 +108,6 @@ public class DataGenerator { } private static void getDataInOneFile(String path, int rowsPerDevice, int num, int humidityDistRadius, int tempDistRadius) throws IOException { - DecimalFormat df = new DecimalFormat("0.0000"); long startTime = dataStartTime; FileWriter fw = new FileWriter(new File(path)); @@ -135,13 +134,13 @@ public class DataGenerator { for (int j = 0; j < rowsPerDevice; ++j) { int humidity = (int) humidityDataGen.next(); - double temp = tempDataGen.next(); + int temp = (int) tempDataGen.next(); int deviceGroup = deviceId % 100; StringBuffer sb = new StringBuffer(); sb.append(deviceId).append(" ").append(tagPrefix).append(deviceId).append(" ").append(deviceGroup) .append(" ").append(dataStartTime).append(" ").append(humidity).append(" ") - .append(df.format(temp)); + .append(temp); bw.write(sb.toString()); bw.write("\n"); diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 287fca7d41..f97075fba8 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -1,6 +1,6 @@ PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) ENDIF () diff --git a/tests/examples/go/src/taosapp/taosapp.go b/tests/examples/go/src/taosapp/taosapp.go index 5c7b92f6fb..f35c902a7a 100644 --- a/tests/examples/go/src/taosapp/taosapp.go +++ b/tests/examples/go/src/taosapp/taosapp.go @@ -19,7 +19,7 @@ import ( "time" "log" "fmt" - _ "taosSql" + _ "github.com/taosdata/driver-go/taosSql" ) func main() { -- GitLab