diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ffe42de2362d3d8a837290799d656b9d32fe96bd..4e05e1ce4c7fea9719e6249bd4b8ed48e5dce4d4 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -45,6 +45,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->pTscObj = pObj; pSql->maxRetry = TSDB_MAX_REPLICA; pSql->fp = fp; + pSql->fetchFp = fp; pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { @@ -159,7 +160,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncError(pSql->fetchFp, param, pRes->code); + tscQueueAsyncRes(pSql); return; } @@ -167,6 +168,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } + tscProcessSql(pSql); } @@ -346,31 +348,32 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { void tscProcessAsyncRes(SSchedMsg *pMsg) { SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; - SSqlCmd *pCmd = &pSql->cmd; +// SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - void *taosres = pSql; +// void *taosres = pSql; // pCmd may be released, so cache pCmd->command - int cmd = pCmd->command; - int code = pRes->code; +// int cmd = pCmd->command; +// int code = pRes->code; // in case of async insert, restore the user specified callback function - bool shouldFree = tscShouldBeFreed(pSql); - - if (cmd == TSDB_SQL_INSERT) { - assert(pSql->fp != NULL); - pSql->fp = pSql->fetchFp; - } - - if (pSql->fp) { - (*pSql->fp)(pSql->param, taosres, code); - } - - if (shouldFree) { - tscDebug("%p sqlObj is automatically freed in async res", pSql); - tscFreeSqlObj(pSql); - } +// bool shouldFree = tscShouldBeFreed(pSql); + +// if (pCmd->command == TSDB_SQL_INSERT) { +// assert(pSql->fp != NULL); + assert(pSql->fp != NULL && pSql->fetchFp != NULL); +// } + +// if (pSql->fp) { + pSql->fp = pSql->fetchFp; + (*pSql->fp)(pSql->param, pSql, pRes->code); +// } + +// if (shouldFree) { +// tscDebug("%p sqlObj is automatically freed in async res", pSql); +// tscFreeSqlObj(pSql); +// } } static void tscProcessAsyncError(SSchedMsg *pMsg) { @@ -420,15 +423,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; + pRes->code = code; if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscQueueAsyncRes(pSql); - return; + tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code)); + goto _error; + } else { + tscDebug("%p get tableMeta successfully", pSql); } - tscDebug("%p get tableMeta successfully", pSql); - if (pSql->pStream == NULL) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -453,11 +456,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); - if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { - return; - } - - goto _error; + // tscProcessSql can add error into async res + tscProcessSql(pSql); + return; } else { // continue to process normal async query if (pCmd->parseFinished) { tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); @@ -481,26 +482,21 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { return; + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; } - if (code == TSDB_CODE_SUCCESS) { - /* - * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, - * and send the required submit block according to index value in supporter to server. - */ - pSql->fp = pSql->fetchFp; // restore the fp - if ((code = tscHandleInsertRetry(pSql)) == TSDB_CODE_SUCCESS) { - return; - } - } - + /* + * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, + * and send the required submit block according to index value in supporter to server. + */ + pSql->fp = pSql->fetchFp; // restore the fp + tscHandleInsertRetry(pSql); } else {// in case of other query type, continue - if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { - return; - } + tscProcessSql(pSql); } - goto _error; + return; } else { tscDebug("%p continue parse sql after get table meta", pSql); @@ -538,7 +534,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { goto _error; } - if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { return; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 06e5ff73bf76d7a2d794fa23544a4967d90d5028..1ec84f023ad74cc8c29654bf14f55781485b691d 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -330,10 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -bool stableQueryFunctChanged(int32_t funcId) { - return (aAggs[funcId].stableFuncId != funcId); -} - /** * the numOfRes should be kept, since it may be used later * and allow the ResultInfo to be re initialized @@ -361,7 +357,6 @@ static bool function_setup(SQLFunctionCtx *pCtx) { } memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResInfo); return true; } @@ -675,16 +670,16 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) { } } -static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t statisRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -697,7 +692,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order != pCtx->param[0].i64Key) { return BLK_DATA_NO_NEEDED; } @@ -709,34 +704,40 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } + + // not initialized yet, it is the first block, load it. + if (pCtx->aOutputBuf == NULL) { + return BLK_DATA_ALL_NEEDED; + } - // result buffer has not been set yet. - return BLK_DATA_ALL_NEEDED; - //todo optimize the filter info -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { // data in current block is not earlier than current result -// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { // data in current block is not earlier than current result + return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } -static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order != pCtx->param[0].i64Key) { return BLK_DATA_NO_NEEDED; } - - return BLK_DATA_ALL_NEEDED; -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { -// return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + + // not initialized yet, it is the first block, load it. + if (pCtx->aOutputBuf == NULL) { + return BLK_DATA_ALL_NEEDED; + } + + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { + return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } ////////////////////////////////////////////////////////////////////////////////////////////// @@ -1549,6 +1550,8 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in * to decide if the value is earlier than current intermediate result */ static void first_dist_function(SQLFunctionCtx *pCtx) { + assert(pCtx->size > 0); + if (pCtx->size == 0) { return; } @@ -1563,7 +1566,12 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } int32_t notNullElems = 0; - + + // data block is discard, not loaded, do not need to check it + if (!pCtx->preAggVals.dataBlockLoaded) { + return; + } + // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -1584,10 +1592,6 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->size == 0) { - return; - } - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -1715,10 +1719,6 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind } static void last_dist_function(SQLFunctionCtx *pCtx) { - if (pCtx->size == 0) { - return; - } - /* * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data @@ -1726,7 +1726,12 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->order != pCtx->param[0].i64Key) { return; } - + + // data block is discard, not loaded, do not need to check it + if (!pCtx->preAggVals.dataBlockLoaded) { + return; + } + int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { @@ -2123,74 +2128,72 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tfree(pData); } -bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval) { - STopBotInfo *pTopBotInfo = (STopBotInfo *)GET_RES_INFO(pCtx)->interResultBuf; - - int32_t numOfExistsRes = pTopBotInfo->num; +/* + * Parameters values: + * 1. param[0]: maximum allowable results + * 2. param[1]: order by type (time or value) + * 3. param[2]: asc/desc order + * + * top/bottom use the intermediate result buffer to keep the intermediate result + */ +static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + // only the first_stage_merge is directly written data into final output buffer + if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + return (STopBotInfo*) pCtx->aOutputBuf; + } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer + return pResInfo->interResultBuf; + } +} + +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { + STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); // required number of results are not reached, continue load data block - if (numOfExistsRes < pCtx->param[0].i64Key) { + if (pTopBotInfo->num < pCtx->param[0].i64Key) { return true; } - tValuePair *pRes = (tValuePair*) pTopBotInfo->res; + tValuePair **pRes = (tValuePair**) pTopBotInfo->res; if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0].v.i64Key; + return GET_INT8_VAL(maxval) > pRes[0]->v.i64Key; case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0].v.i64Key; + return GET_INT16_VAL(maxval) > pRes[0]->v.i64Key; case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0].v.i64Key; + return GET_INT32_VAL(maxval) > pRes[0]->v.i64Key; case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0].v.i64Key; + return GET_INT64_VAL(maxval) > pRes[0]->v.i64Key; case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0].v.dKey; + return GET_FLOAT_VAL(maxval) > pRes[0]->v.dKey; case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0].v.dKey; + return GET_DOUBLE_VAL(maxval) > pRes[0]->v.dKey; default: return true; } } else { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0].v.i64Key; + return GET_INT8_VAL(minval) < pRes[0]->v.i64Key; case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0].v.i64Key; + return GET_INT16_VAL(minval) < pRes[0]->v.i64Key; case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0].v.i64Key; + return GET_INT32_VAL(minval) < pRes[0]->v.i64Key; case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0].v.i64Key; + return GET_INT64_VAL(minval) < pRes[0]->v.i64Key; case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0].v.dKey; + return GET_FLOAT_VAL(minval) < pRes[0]->v.dKey; case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0].v.dKey; + return GET_DOUBLE_VAL(minval) < pRes[0]->v.dKey; default: return true; } } } -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (STopBotInfo*) pCtx->aOutputBuf; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return pResInfo->interResultBuf; - } -} - /* * keep the intermediate results during scan data blocks in the format of: * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ @@ -3376,7 +3379,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = pResInfo->interResultBuf; - int32_t numOfElems = pCtx->size; + int32_t numOfElems = 0; // todo : opt with pre-calculated result // column missing cause the hasNull to be true @@ -4412,7 +4415,7 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... * */ -int32_t funcCompatDefList[] = { +int32_t functionCompatList[] = { // count, sum, avg, min, max, stddev, percentile, apercentile, first, last 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z @@ -4451,7 +4454,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, sum_func_merge, sum_func_second_merge, - precal_req_load_info, + statisRequired, }, { // 2 @@ -4466,7 +4469,7 @@ SQLAggFuncElem aAggs[] = {{ avg_finalizer, avg_func_merge, avg_func_second_merge, - precal_req_load_info, + statisRequired, }, { // 3 @@ -4481,7 +4484,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, min_func_merge, min_func_second_merge, - precal_req_load_info, + statisRequired, }, { // 4 @@ -4496,7 +4499,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, max_func_merge, max_func_second_merge, - precal_req_load_info, + statisRequired, }, { // 5 @@ -4511,7 +4514,7 @@ SQLAggFuncElem aAggs[] = {{ stddev_finalizer, noop1, noop1, - data_req_load_info, + dataBlockRequired, }, { // 6 @@ -4526,7 +4529,7 @@ SQLAggFuncElem aAggs[] = {{ percentile_finalizer, noop1, noop1, - data_req_load_info, + dataBlockRequired, }, { // 7 @@ -4541,7 +4544,7 @@ SQLAggFuncElem aAggs[] = {{ apercentile_finalizer, apercentile_func_merge, apercentile_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 8 @@ -4556,7 +4559,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, noop1, noop1, - first_data_req_info, + firstFuncRequired, }, { // 9 @@ -4571,7 +4574,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, noop1, noop1, - last_data_req_info, + lastFuncRequired, }, { // 10 @@ -4587,7 +4590,7 @@ SQLAggFuncElem aAggs[] = {{ last_row_finalizer, noop1, last_dist_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 11 @@ -4603,7 +4606,7 @@ SQLAggFuncElem aAggs[] = {{ top_bottom_func_finalizer, top_func_merge, top_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 12 @@ -4619,7 +4622,7 @@ SQLAggFuncElem aAggs[] = {{ top_bottom_func_finalizer, bottom_func_merge, bottom_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 13 @@ -4649,7 +4652,7 @@ SQLAggFuncElem aAggs[] = {{ twa_function_finalizer, twa_func_merge, twa_function_copy, - data_req_load_info, + dataBlockRequired, }, { // 15 @@ -4664,7 +4667,7 @@ SQLAggFuncElem aAggs[] = {{ leastsquares_finalizer, noop1, noop1, - data_req_load_info, + dataBlockRequired, }, { // 16 @@ -4694,7 +4697,7 @@ SQLAggFuncElem aAggs[] = {{ doFinalizer, copy_function, copy_function, - data_req_load_info, + dataBlockRequired, }, { // 18 @@ -4724,7 +4727,7 @@ SQLAggFuncElem aAggs[] = {{ ts_comp_finalize, copy_function, copy_function, - data_req_load_info, + dataBlockRequired, }, { // 20 @@ -4754,7 +4757,7 @@ SQLAggFuncElem aAggs[] = {{ doFinalizer, copy_function, copy_function, - data_req_load_info, + dataBlockRequired, }, { // 22, multi-output, tag function has only one result @@ -4784,7 +4787,7 @@ SQLAggFuncElem aAggs[] = {{ doFinalizer, copy_function, copy_function, - data_req_load_info, + dataBlockRequired, }, { // 24 @@ -4799,7 +4802,7 @@ SQLAggFuncElem aAggs[] = {{ doFinalizer, noop1, noop1, - data_req_load_info, + dataBlockRequired, }, // distributed version used in two-stage aggregation processes { @@ -4815,7 +4818,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, first_dist_func_merge, first_dist_func_second_merge, - first_dist_data_req_info, + firstDistFuncRequired, }, { // 26 @@ -4830,7 +4833,7 @@ SQLAggFuncElem aAggs[] = {{ function_finalizer, last_dist_func_merge, last_dist_func_second_merge, - last_dist_data_req_info, + lastDistFuncRequired, }, { // 27 @@ -4845,7 +4848,7 @@ SQLAggFuncElem aAggs[] = {{ doFinalizer, noop1, copy_function, - data_req_load_info, + dataBlockRequired, }, { // 28 @@ -4860,7 +4863,7 @@ SQLAggFuncElem aAggs[] = {{ rate_finalizer, rate_func_merge, rate_func_copy, - data_req_load_info, + dataBlockRequired, }, { // 29 @@ -4875,7 +4878,7 @@ SQLAggFuncElem aAggs[] = {{ rate_finalizer, rate_func_merge, rate_func_copy, - data_req_load_info, + dataBlockRequired, }, { // 30 @@ -4890,7 +4893,7 @@ SQLAggFuncElem aAggs[] = {{ sumrate_finalizer, sumrate_func_merge, sumrate_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 31 @@ -4905,7 +4908,7 @@ SQLAggFuncElem aAggs[] = {{ sumrate_finalizer, sumrate_func_merge, sumrate_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 32 @@ -4920,7 +4923,7 @@ SQLAggFuncElem aAggs[] = {{ sumrate_finalizer, sumrate_func_merge, sumrate_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 33 @@ -4935,7 +4938,7 @@ SQLAggFuncElem aAggs[] = {{ sumrate_finalizer, sumrate_func_merge, sumrate_func_second_merge, - data_req_load_info, + dataBlockRequired, }, { // 34 @@ -4950,5 +4953,5 @@ SQLAggFuncElem aAggs[] = {{ noop1, noop1, noop1, - data_req_load_info, + dataBlockRequired, }}; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ca6dfcee9efb03058ad2401ed57f2dce6f581d59..3bcbe37b81055a609c88e06429002a96f2e4df1e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2471,7 +2471,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { startIdx++; } - int32_t factor = funcCompatDefList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; + int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions @@ -2489,7 +2489,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { continue; } - if (funcCompatDefList[functionId] != factor) { + if (functionCompatList[functionId] != factor) { return false; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 88fcc3828ec8756a4ac8926bb03fee8712103ee7..564d5ae23f75a22a36d6538e56ef5a0f4655f162 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -339,7 +339,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { } if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? pRes->numOfRows: pRes->code; + rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS)? pRes->numOfRows: pRes->code; bool shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, pSql, rpcMsg->code); @@ -412,7 +412,7 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; //? + pSql->ipList = tscMgmtIpSet; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -476,6 +476,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t vgIndex = pTableMetaInfo->vgroupIndex; SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -549,6 +551,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char assert(index >= 0); if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { + assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); @@ -1372,7 +1375,6 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); pRes->code = TSDB_CODE_SUCCESS; - if (pRes->rspType == 0) { pRes->numOfRows = numOfRes; pRes->row = 0; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 73b93435ad1e8224707bdcf6f029d77ee4d51737..8048a2389b150801d2f34c7bccd9ca11e474fd19 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -481,25 +481,14 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) { if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) && (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH) && - (pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { + pCmd->command == TSDB_SQL_SHOW || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_FETCH) && + (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]); + tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); tscProcessSql(pSql); - - // in case of sync model query, waits for response and then goes on -// if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) { -// sem_wait(&pSql->rspSem); - -// tscFreeSqlObj(pSql); -// tscDebug("%p sqlObj is freed by app", pSql); -// } else { - tscDebug("%p sqlObj will be freed while rsp received", pSql); -// } - return true; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 5e2ba46fbcca85f4e9e758ce0f655cc0b3accddf..dd4ed991ed9845b5fdabc3a0e07727bfcd1756ef 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1895,9 +1895,11 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) { assert(pSupporter->index < pSupporter->pState->numOfTotal); STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); - pRes->code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); - if (pRes->code != TSDB_CODE_SUCCESS) { - return pRes->code; + int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); + + if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; // here the pSql may have been released already. } return tscProcessSql(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index eea5d60f64cedd0442d69ac96f21bc920bae2c90..3e0fe0b4bec2c392c48fcbec84508ec5b1d3303a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1648,6 +1648,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm } pNew->fp = fp; + pNew->fetchFp = fp; pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; @@ -1803,6 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } pNew->fp = fp; + pNew->fetchFp = fp; + pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; @@ -2005,7 +2008,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; - while (++pTableMetaInfo->vgroupIndex < totalVgroups) { + if (++pTableMetaInfo->vgroupIndex < totalVgroups) { tscDebug("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql, pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal); @@ -2041,11 +2044,9 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { // set the callback function pSql->fp = fp; - int32_t ret = tscProcessSql(pSql); - if (ret == TSDB_CODE_SUCCESS) { - return; - } else {// todo check for failure - } + tscProcessSql(pSql); + } else { + tscDebug("%p try all %d vnodes, query complete. current numOfRes:%" PRId64, pSql, totalVgroups, pRes->numOfClauseTotal); } } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 971bd0a110781f17b88c278b61fd98558cff2627..4eba81e29d6e365b7be4000e0c57ad9dfe1a1501 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -70,7 +70,13 @@ int32_t main(int32_t argc, char *argv[]) { } #endif #ifdef TAOS_RANDOM_FILE_FAIL - else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { + else if (strcmp(argv[i], "--random-file-fail-output") == 0) { + if ((i < argc - 1) && (argv[i + 1][0] != '-')) { + taosSetRandomFileFailOutput(argv[++i]); + } else { + taosSetRandomFileFailOutput(NULL); + } + } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { if ( (i+1) < argc ) { int factor = atoi(argv[i+1]); printf("The factor of random failure is %d\n", factor); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 1faad6b141733b917893e02e10bfd7fa4eaff2d9..cdf5f9612cf192cc5ba445f232e408531cfc3afa 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -121,6 +121,7 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; + uint64_t ioTime; uint64_t computTime; } SQueryCostInfo; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 9b818b367f3b448c70ebb04e0df6a57071195603..8fe3c0f4953fe02e5de6c4bced125d164141f086 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -37,7 +37,7 @@ SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot); #define curTimeWindow(_winres) ((_winres)->curIndex) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize); +int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize); char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 0c69bc02d39495cc6b77f00bd28524036cfef211..e57cb2645666af6887dc3253a76ce603314afbdf 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -125,7 +125,8 @@ typedef struct SArithmeticSupport { } SArithmeticSupport; typedef struct SQLPreAggVal { - bool isSet; + bool isSet; // statistics info set or not + bool dataBlockLoaded; // data block is loaded or not SDataStatis statis; } SQLPreAggVal; @@ -224,25 +225,14 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) #define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) -/* - * the status of one block, used in metric query. all blocks are mixed together, - * we need the status to decide if one block is a first/end/inter block of one meter - */ -enum { - BLK_FILE_BLOCK = 0x1, - BLK_BLOCK_LOADED = 0x2, - BLK_CACHE_BLOCK = 0x4, // in case of cache block, block must be loaded -}; - /* determine the real data need to calculated the result */ enum { - BLK_DATA_NO_NEEDED = 0x0, + BLK_DATA_NO_NEEDED = 0x0, BLK_DATA_STATIS_NEEDED = 0x1, - BLK_DATA_ALL_NEEDED = 0x3, + BLK_DATA_ALL_NEEDED = 0x3, + BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter }; -#define SET_DATA_BLOCK_NOT_LOADED(x) ((x) &= (~BLK_BLOCK_LOADED)); - typedef struct STwaInfo { TSKEY lastKey; int8_t hasResult; // flag to denote has value @@ -264,12 +254,9 @@ typedef struct STwaInfo { /* global sql function array */ extern struct SQLAggFuncElem aAggs[]; -/* compatible check array list */ -extern int32_t funcCompatDefList[]; - -bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval); +extern int32_t functionCompatList[]; // compatible check array list -bool stableQueryFunctChanged(int32_t funcId); +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); void resetResultInfo(SResultInfo *pResInfo); void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d1b336fb4b1a0e57b367be5654825cfb7bc8707b..37defc3ccf4b961c090c69c5f0785927a18160bc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1358,6 +1358,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.isSet = false; } + pCtx->preAggVals.dataBlockLoaded = (inputData != NULL); + // limit/offset query will affect this value pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0; pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; @@ -1928,73 +1930,46 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi pQuery->pSelectExpr[columnIndex].bytes * realRowId; } -/** - * decrease the refcount for each table involved in this query - * @param pQInfo - */ -UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) { - if (pQInfo != NULL) { - // assert(taosHashGetSize(pQInfo->tableqinfoGroupInfo) >= 1); - } - -#if 0 - if (pQInfo == NULL || pQInfo->tableqinfoGroupInfo.numOfTables == 1) { - atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1); - qDebug("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode, - pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries); - } else { - int32_t num = 0; - for (int32_t i = 0; i < pQInfo->tableqinfoGroupInfo.numOfTables; ++i) { - SMeterObj *pMeter = getMeterObj(pQInfo->tableqinfoGroupInfo, pQInfo->pSidSet->pTableIdList[i]->sid); - atomic_fetch_sub_32(&(pMeter->numOfQueries), 1); - - if (pMeter->numOfQueries > 0) { - qDebug("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pMeter->vnode, pMeter->sid, - pMeter->meterId, pMeter->numOfQueries); - num++; - } - } +#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) - /* - * in order to reduce log output, for all meters of which numOfQueries count are 0, - * we do not output corresponding information - */ - num = pQInfo->tableqinfoGroupInfo.numOfTables - num; - qDebug("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo, - pQInfo->tableqinfoGroupInfo.numOfTables, num); - } -#endif -} - -static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, - int32_t numOfTotalPoints) { - if (pDataStatis == NULL) { +static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, + int32_t numOfRows) { + SQuery* pQuery = pRuntimeEnv->pQuery; + if (pDataStatis == NULL || (pQuery->numOfFilterCols == 0 && (!pRuntimeEnv->topBotQuery))) { return true; } -#if 0 for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - int32_t colIndex = pFilterInfo->info.colIndex; - // this column not valid in current data block - if (colIndex < 0 || pDataStatis[colIndex].colId != pFilterInfo->info.data.colId) { - continue; + int32_t index = -1; + for(int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (pDataStatis[i].colId == pFilterInfo->info.colId) { + index = i; + break; + } + } + + // no statistics data + if (index == -1) { + return true; } // not support pre-filter operation on binary/nchar data type - if (!vnodeSupportPrefilter(pFilterInfo->info.data.type)) { - continue; + if (!IS_PREFILTER_TYPE(pFilterInfo->info.type)) { + return true; } // all points in current column are NULL, no need to check its boundary value - if (pDataStatis[colIndex].numOfNull == numOfTotalPoints) { + if (pDataStatis[index].numOfNull == numOfRows) { continue; } - if (pFilterInfo->info.info.type == TSDB_DATA_TYPE_FLOAT) { - float minval = *(double *)(&pDataStatis[colIndex].min); - float maxval = *(double *)(&pDataStatis[colIndex].max); + SDataStatis* pDataBlockst = &pDataStatis[index]; + + if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) { + float minval = *(double *)(&pDataBlockst->min); + float maxval = *(double *)(&pDataBlockst->max); for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval)) { @@ -2003,53 +1978,50 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun } } else { for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataStatis[colIndex].min, - (char *)&pDataStatis[colIndex].max)) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max)) { return true; } } } } - // todo disable this opt code block temporarily - // for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - // int32_t functId = pQuery->pSelectExpr[i].base.functionId; - // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); - // } - // } + if (pRuntimeEnv->topBotQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return topbot_datablock_filter(&pCtx[i], functionId, (char *)&pDataStatis[i].min, (char *)&pDataStatis[i].max); + } + } + } -#endif - return true; + return false; } -SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis) { +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - uint32_t r = 0; - SArray * pDataBlock = NULL; - + uint32_t status = 0; if (pQuery->numOfFilterCols > 0) { - r = BLK_DATA_ALL_NEEDED; - } else { - // check if this data block is required to load + status = BLK_DATA_ALL_NEEDED; + } else { // check if this data block is required to load for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base; int32_t functionId = pSqlFunc->functionId; int32_t colId = pSqlFunc->colInfo.colId; - r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); + status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); } if (pRuntimeEnv->pTSBuf > 0 || QUERY_IS_INTERVAL_QUERY(pQuery)) { - r |= BLK_DATA_ALL_NEEDED; + status |= BLK_DATA_ALL_NEEDED; } } - if (r == BLK_DATA_NO_NEEDED) { - qDebug("QInfo:%p data block discard, rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->rows); + if (status == BLK_DATA_NO_NEEDED) { + qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pRuntimeEnv->summary.discardBlocks += 1; - } else if (r == BLK_DATA_STATIS_NEEDED) { + } else if (status == BLK_DATA_STATIS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } @@ -2057,32 +2029,34 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pRuntimeEnv->summary.loadBlockStatis += 1; if (*pStatis == NULL) { // data block statistics does not exist, load data block - pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); + *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { - assert(r == BLK_DATA_ALL_NEEDED); + assert(status == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter - pRuntimeEnv->summary.loadBlockStatis +=1; + pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - if (!needToLoadDataBlock(pQuery,*pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) qDebug("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv)); #endif // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; - // return DISK_DATA_DISCARDED; + qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + return BLK_DATA_DISCARD; } pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; - pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); + *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); } - return pDataBlock; + return TSDB_CODE_SUCCESS; } int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { @@ -2225,13 +2199,13 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->order.order); TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -2259,7 +2233,11 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { ensureOutputBuffer(pRuntimeEnv, &blockInfo); SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); + SArray *pDataBlock = NULL; + if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { + pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; + continue; + } // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1; @@ -2282,8 +2260,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { -// int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); // removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step); pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window @@ -2978,16 +2954,23 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize) { +int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize) { int32_t numOfCols = pQuery->numOfOutput; pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); + if (pResultRow->resultInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } pResultRow->pos = *posInfo; char* buf = calloc(1, interBufSize); + if (buf == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } // set the intermediate result output buffer setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); + return TSDB_CODE_SUCCESS; } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3368,7 +3351,10 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { int32_t initialSize = 16; int32_t initialThreshold = 100; - initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } } else { // in other aggregate query, do not initialize the windowResInfo } @@ -3383,12 +3369,10 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo, numOfCols); } -#define SET_CURRENT_QUERY_TABLE_INFO(_runtime, _tableInfo) \ - do { \ - SQuery *_query = (_runtime)->pQuery; \ - _query->current = _tableInfo; \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_query)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_query))); \ +#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ + do { \ + assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ + (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ } while (0) /** @@ -3700,7 +3684,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, +static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; @@ -3859,9 +3843,10 @@ static void queryCostStatis(SQInfo *pQInfo) { // pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0, // pSummary->skippedFileBlocks, pSummary->totalGenData); - qDebug("QInfo:%p :cost summary: elpased time:%"PRId64" us, total blocks:%d, use block statis:%d, use block data:%d, " - "total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, - pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, io time:%"PRId64" us, total blocks:%d, load block statis:%d," + " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->ioTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); // qDebug("QInfo:%p cost: temp file:%d Bytes", pQInfo, pSummary->tmpBufferInDisk); // @@ -4189,7 +4174,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_INT; // group id } - initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { @@ -4206,7 +4194,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { @@ -4237,6 +4228,23 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } +static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTableQueryInfo, SDataBlockInfo* pBlockInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { + setExecutionContext(pQInfo, pTableQueryInfo->groupIndex, pBlockInfo->window.ekey + step); + } else { // interval query + TSKEY nextKey = pBlockInfo->window.skey; + setIntervalQueryRange(pQInfo, nextKey); + + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { + setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); + } + } +} + static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; @@ -4247,10 +4255,12 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; + if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -4260,24 +4270,19 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { break; } - assert(*pTableQueryInfo != NULL); - SET_CURRENT_QUERY_TABLE_INFO(pRuntimeEnv, *pTableQueryInfo); + pQuery->current = *pTableQueryInfo; + CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + + if (!pRuntimeEnv->groupbyNormalCol) { + setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); + } SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); + SArray *pDataBlock = NULL; - if (!pRuntimeEnv->groupbyNormalCol) { - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1; - setExecutionContext(pQInfo, (*pTableQueryInfo)->groupIndex, blockInfo.window.ekey + step); - } else { // interval query - TSKEY nextKey = blockInfo.window.skey; - setIntervalQueryRange(pQInfo, nextKey); - - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { - setAdditionalInfo(pQInfo, (*pTableQueryInfo)->pTable, *pTableQueryInfo); - } - } + if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { + pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; + continue; } summary->totalRows += blockInfo.rows; @@ -4537,7 +4542,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) { if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5035,6 +5039,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); + } // record the total elapsed time @@ -5366,7 +5371,7 @@ static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTable qDebug("qmsg:%p create arithmetic expr from binary string: %s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); tExprNode* pExprNode = NULL; - TRY(32) { + TRY(TSDB_MAX_TAGS) { pExprNode = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes); } CATCH( code ) { CLEANUP_EXECUTE(); @@ -5738,6 +5743,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, size_t s = taosArrayGetSize(pa); SArray* p1 = taosArrayInit(s, POINTER_BYTES); + if (p1 == NULL) { + goto _cleanup; + } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { void* pTable = taosArrayGetP(pa, j); @@ -5752,13 +5761,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, void* buf = pQInfo->pBuf + index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, pTable, window, buf); + if (item == NULL) { + goto _cleanup; + } item->groupIndex = i; taosArrayPush(p1, &item); taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); index += 1; } - - taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); } pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); @@ -6155,11 +6165,6 @@ _over: //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; if (code != TSDB_CODE_SUCCESS) { *pQInfo = NULL; - } else { -// SQInfo* pq = (SQInfo*) (*pQInfo); - -// T_REF_INC(pq); -// T_REF_INC(pq); } // if failed to add ref for all meters in this query, abort current query diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index dce2c24ea007e656398eb3ccef483a5233589946..acdc46fcc1dc867a2b391326da8fa814722c9282 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -50,9 +50,15 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + if (pWindowResInfo->pResult == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { SPosInfo posInfo = {-1, -1}; - createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize); + int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -76,9 +82,11 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) { return; } - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; - destroyTimeWindowRes(pResult, numOfCols); + if (pWindowResInfo->pResult != NULL) { + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + destroyTimeWindowRes(pResult, numOfCols); + } } taosHashCleanup(pWindowResInfo->hashList); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 9e5da903b1402fa048e40a07666c19530e73c105..070e6e7ac55aad27b34512c37f1395bdf0a97437 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1801,7 +1801,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL); - + + // todo opt perf size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle); for(int32_t i = 0; i < numOfCols; ++i) { SDataStatis* st = &pHandle->statis[i]; @@ -1820,6 +1821,13 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } + + // todo opt perf + SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i); + if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) { + pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst; + pHandle->statis[i].max = pBlockInfo->compBlock->keyLast; + } } return TSDB_CODE_SUCCESS; @@ -2193,7 +2201,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT int32_t ret = TSDB_CODE_SUCCESS; tExprNode* expr = NULL; - TRY(32) { + TRY(TSDB_MAX_TAGS) { expr = exprTreeFromTableName(tbnameCond); if (expr == NULL) { expr = exprTreeFromBinary(pTagCond, len); diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 04e500743c1c64d1ec6f636d92cc8904d7a4a2a2..de52a409674b9518fa908bd8131024c5b1c58010 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -19,13 +19,14 @@ #ifdef TAOS_RANDOM_FILE_FAIL void taosSetRandomFileFailFactor(int factor); -ssize_t taos_tread(int fd, void *buf, size_t count); -ssize_t taos_twrite(int fd, void *buf, size_t count); -off_t taos_lseek(int fd, off_t offset, int whence); +void taosSetRandomFileFailOutput(const char *path); +ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line); +ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line); +off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line); -#define tread(fd, buf, count) taos_tread(fd, buf, count) -#define twrite(fd, buf, count) taos_twrite(fd, buf, count) -#define lseek(fd, offset, whence) taos_lseek(fd, offset, whence) +#define tread(fd, buf, count) taos_tread(fd, buf, count, __FILE__, __LINE__) +#define twrite(fd, buf, count) taos_twrite(fd, buf, count, __FILE__, __LINE__) +#define lseek(fd, offset, whence) taos_lseek(fd, offset, whence, __FILE__, __LINE__) #endif // TAOS_RANDOM_FILE_FAIL diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index d3c622633de7cdebfe43a7492f7a882823dec0e2..df63d567c795c69b031c6ab97112a1b51d0c2e94 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -413,57 +413,90 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { *data = NULL; // note: extend lifespan before dec ref count - if (pCacheObj->extendLifespan) { + bool inTrashCan = pNode->inTrashCan; + + if (pCacheObj->extendLifespan && (!inTrashCan)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); uDebug("cache:%s data:%p extend life time to %"PRId64 " before release", pCacheObj->name, pNode->data, pNode->expireTime); } - bool inTrashCan = pNode->inTrashCan; - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1); - - // NOTE: once refcount is decrease, pNode may be free by other thread immediately. - int32_t ref = T_REF_DEC(pNode); - - if (inTrashCan) { - // Remove it if the ref count is 0. - // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when - // the node is in trashcan. - if (ref == 0) { - __cache_wr_lock(pCacheObj); - assert(pNode->pTNodeHeader->pData == pNode); - taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); - __cache_unlock(pCacheObj); + if (_remove) { + __cache_wr_lock(pCacheObj); + + // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. + int32_t ref = T_REF_DEC(pNode); + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); + + /* + * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users + * releasing this resources. + * + * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread + * that tries to do the same thing. + */ + if (pNode->inTrashCan) { + if (ref == 0) { + assert(pNode->pTNodeHeader->pData == pNode); + taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); + } + } else { + if (ref > 0) { + assert(pNode->pTNodeHeader == NULL); + taosCacheMoveToTrash(pCacheObj, pNode); + } else { + taosCacheReleaseNode(pCacheObj, pNode); + } } + __cache_unlock(pCacheObj); + } else { - assert(pNode->pTNodeHeader == NULL); - - if (_remove) { // not in trash can, but need to remove it - __cache_wr_lock(pCacheObj); - - /* - * If not referenced by other users. Otherwise move this node to trashcan wait for all users - * releasing this resources. - * - * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread - * that tries to do the same thing. - */ + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1); + + // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. + int32_t ref = T_REF_DEC(pNode); + + if (inTrashCan) { + // Remove it if the ref count is 0. + // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when + // the node is in trashcan. if (ref == 0) { - if (T_REF_VAL_GET(pNode) == 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { - taosCacheMoveToTrash(pCacheObj, pNode); - } + __cache_wr_lock(pCacheObj); + assert(pNode->pTNodeHeader->pData == pNode); + taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); + __cache_unlock(pCacheObj); } - __cache_unlock(pCacheObj); -// } else { // extend its life time -// if (pCacheObj->extendLifespan) { -// atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); -// uDebug("cache:%s data:%p extend life time to %"PRId64 " after release", pCacheObj->name, pNode->data, pNode->expireTime); -// } } } + +// else { +// if (_remove) { // not in trash can, but need to remove it +// __cache_wr_lock(pCacheObj); +// +// /* +// * If not referenced by other users. Otherwise move this node to trashcan wait for all users +// * releasing this resources. +// * +// * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread +// * that tries to do the same thing. +// */ +// if (ref == 0) { +// if (T_REF_VAL_GET(pNode) == 0) { +// taosCacheReleaseNode(pCacheObj, pNode); +// } else { +// taosCacheMoveToTrash(pCacheObj, pNode); +// } +// } else if (ref > 0) { +// if (!pNode->inTrashCan) { +// assert(pNode->pTNodeHeader == NULL); +// taosCacheMoveToTrash(pCacheObj, pNode); +// } +// } +// +// __cache_unlock(pCacheObj); +// } +// } } void taosCacheEmpty(SCacheObj *pCacheObj) { diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 92eeaef1262cec6d3ee5c5d09ba0c02b4b2b097d..d1a9c2461518a787920eac6de591b320a265d20d 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -29,14 +29,55 @@ #ifdef TAOS_RANDOM_FILE_FAIL static int random_file_fail_factor = 20; +static FILE *fpRandomFileFailOutput = NULL; void taosSetRandomFileFailFactor(int factor) { random_file_fail_factor = factor; } + +static void close_random_file_fail_output() +{ + if (fpRandomFileFailOutput != NULL) { + if (fpRandomFileFailOutput != stdout) { + fclose(fpRandomFileFailOutput); + } + fpRandomFileFailOutput = NULL; + } +} + +static void random_file_fail_output_sig(int sig) +{ + fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig); + + struct sigaction act = {0}; + act.sa_handler = SIG_DFL; + sigaction(sig, &act, NULL); + + close_random_file_fail_output(); + exit(EXIT_FAILURE); +} + +void taosSetRandomFileFailOutput(const char *path) +{ + if (path == NULL) { + fpRandomFileFailOutput = stdout; + } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) { + atexit(close_random_file_fail_output); + } else { + printf("failed to open random file fail log file '%s', errno=%d\n", path, errno); + return; + } + + struct sigaction act = {0}; + act.sa_handler = random_file_fail_output_sig; + sigaction(SIGFPE, &act, NULL); + sigaction(SIGSEGV, &act, NULL); + sigaction(SIGILL, &act, NULL); +} #endif -ssize_t taos_tread(int fd, void *buf, size_t count) +ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line) { #ifdef TAOS_RANDOM_FILE_FAIL if (random_file_fail_factor > 0) { @@ -49,7 +90,7 @@ ssize_t taos_tread(int fd, void *buf, size_t count) return tread(fd, buf, count); } -ssize_t taos_twrite(int fd, void *buf, size_t count) +ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line) { #ifdef TAOS_RANDOM_FILE_FAIL if (random_file_fail_factor > 0) { @@ -62,7 +103,7 @@ ssize_t taos_twrite(int fd, void *buf, size_t count) return twrite(fd, buf, count); } -off_t taos_lseek(int fd, off_t offset, int whence) +off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line) { #ifdef TAOS_RANDOM_FILE_FAIL if (random_file_fail_factor > 0) { diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index ff58e219b07fba049a36b14fc97446010499321d..987783711b2452b17c56568fe4c63ed506c51e30 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -110,6 +110,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle == NULL) { // failed to register qhandle pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; qDestroyQueryInfo(pQInfo); // destroy it directly + vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void*) pQInfo, tstrerror(pRsp->code)); } else { assert(*handle == pQInfo); pRsp->qhandle = htobe64((uint64_t) pQInfo); @@ -125,12 +126,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { assert(pQInfo == NULL); } + if (handle != NULL) { + vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle); + dnodePutItemIntoReadQueue(pVnode, *handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } - vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo); } else { assert(pCont != NULL); handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); @@ -138,12 +141,13 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); code = TSDB_CODE_QRY_INVALID_QHANDLE; } else { - vDebug("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, (void*) pCont); + vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); code = TSDB_CODE_VND_ACTION_IN_PROGRESS; qTableQuery(*handle); // do execute query } qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } + return code; } @@ -155,7 +159,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->free = htons(pRetrieve->free); - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, *(void**) pRetrieve->qhandle); + vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle); memset(pRet, 0, sizeof(SRspRet)); diff --git a/tests/script/unique/cluster/client3.sim b/tests/script/unique/cluster/client3.sim new file mode 100644 index 0000000000000000000000000000000000000000..4b3024f881864cdce103afaca8f7b2925ad564d3 --- /dev/null +++ b/tests/script/unique/cluster/client3.sim @@ -0,0 +1,55 @@ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +loop_lable: +print ====================== client3 start loop: dynamic create table and insert data, select, drop table ............ + +$totalRows = 0 + +#sql create table $stb ( ts timestamp, c1 int) tags ( t1 int, t2 binary(16) ) +$tagBinary = ' . client3 +$tagBinary = $tagBinary . ' + +$rowsPerLoop = 100 +$ts = $tsStart + +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb using $stb tags ( $i , $tagBinary ) values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x +# print ====== app insert totalRows: $totalRows + $i = $i + 1 +endw + +sql select count(*) from $stb where t2 == $tagBinary +if $data00 != $totalRows then + print data00 $data00 totalRows $totalRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client3 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +endi + +print ====================== client3 drop table +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + sql drop table if exists $tb + $i = $i + 1 +endw +goto loop_lable \ No newline at end of file diff --git a/tests/script/unique/cluster/client4.sim b/tests/script/unique/cluster/client4.sim new file mode 100644 index 0000000000000000000000000000000000000000..a4c347695ea4f1adae1dfd7f3a6909533fa64381 --- /dev/null +++ b/tests/script/unique/cluster/client4.sim @@ -0,0 +1,113 @@ +$tblStart = 0 +$tblEnd = 2000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +loop_lable: +print ====================== client4 start loop: create db2, tables and insert data, select, drop db2 ............ + +$db = db2 +$stb = stb2 + +print create database if not exists $db replica 2 +sql create database if not exists $db replica 2 +sql use $db + +print ==== client4start create table +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + sql create table $tb (ts timestamp, c1 int) + $i = $i + 1 +endw + + +print ==== client4start insert, include multi table data in one insert sql +$totalRows = 0 +$totalRowsPerTbl = 0 + +$rowsPerLoop = 100 +$ts = $tsStart + +$i = $tblStart +while $i < $tblEnd + $tb0 = dtb . $i + $i = $i + 1 + $tb1 = dtb . $i + $i = $i + 1 + $tb2 = dtb . $i + $i = $i + 1 + $tb3 = dtb . $i + $i = $i + 1 + $tb4 = dtb . $i + $i = $i + 1 + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRowsPerTbl = $totalRowsPerTbl + $x + $x = $x * 5 + $totalRows = $totalRows + $x +endw + +sql select count(*) from tb10 +if $data00 != $totalRowsPerTbl then + print data00 $data00 totalRowsPerTbl $totalRowsPerTbl + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client4 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +endi + + +print ====client4 start alter table +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + sql alter table $tb add c2 float + $i = $i + 1 +endw + +print ====client4 continue insert, include multi table data in one insert sql + +$i = $tblStart +while $i < $tblEnd + $tb0 = dtb . $i + $i = $i + 1 + $tb1 = dtb . $i + $i = $i + 1 + $tb2 = dtb . $i + $i = $i + 1 + $tb3 = dtb . $i + $i = $i + 1 + $tb4 = dtb . $i + $i = $i + 1 + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRowsPerTbl = $totalRowsPerTbl + $x + $x = $x * 5 + $totalRows = $totalRows + $x +endw + +sql select count(*) from tb10 +if $data00 != $totalRowsPerTbl then + print data00 $data00 totalRowsPerTbl $totalRowsPerTbl + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client4 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +endi + +print ====================== client4 drop database +sql drop if exists database $db +goto loop_lable \ No newline at end of file diff --git a/tests/script/unique/cluster/client5.sim b/tests/script/unique/cluster/client5.sim new file mode 100644 index 0000000000000000000000000000000000000000..5960b28fdda98b330c0a106716dcc45ab05757d5 --- /dev/null +++ b/tests/script/unique/cluster/client5.sim @@ -0,0 +1,113 @@ +$tblStart = 0 +$tblEnd = 2000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +loop_lable: +print ====================== client5 start loop query + +$db = db2 +$stb = stb2 + +print create database if not exists $db replica 2 +sql create database if not exists $db replica 2 +sql use $db + +print ==== client4start create table +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + sql create table $tb (ts timestamp, c1 int) + $i = $i + 1 +endw + + +print ==== client4start insert, include multi table data in one insert sql +$totalRows = 0 +$totalRowsPerTbl = 0 + +$rowsPerLoop = 100 +$ts = $tsStart + +$i = $tblStart +while $i < $tblEnd + $tb0 = dtb . $i + $i = $i + 1 + $tb1 = dtb . $i + $i = $i + 1 + $tb2 = dtb . $i + $i = $i + 1 + $tb3 = dtb . $i + $i = $i + 1 + $tb4 = dtb . $i + $i = $i + 1 + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRowsPerTbl = $totalRowsPerTbl + $x + $x = $x * 5 + $totalRows = $totalRows + $x +endw + +sql select count(*) from tb10 +if $data00 != $totalRowsPerTbl then + print data00 $data00 totalRowsPerTbl $totalRowsPerTbl + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client4 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +endi + + +print ====client4 start alter table +$i = $tblStart +while $i < $tblEnd + $tb = dtb . $i + sql alter table $tb add c2 float + $i = $i + 1 +endw + +print ====client4 continue insert, include multi table data in one insert sql + +$i = $tblStart +while $i < $tblEnd + $tb0 = dtb . $i + $i = $i + 1 + $tb1 = dtb . $i + $i = $i + 1 + $tb2 = dtb . $i + $i = $i + 1 + $tb3 = dtb . $i + $i = $i + 1 + $tb4 = dtb . $i + $i = $i + 1 + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRowsPerTbl = $totalRowsPerTbl + $x + $x = $x * 5 + $totalRows = $totalRows + $x +endw + +sql select count(*) from tb10 +if $data00 != $totalRowsPerTbl then + print data00 $data00 totalRowsPerTbl $totalRowsPerTbl + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client4 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +endi + +print ====================== client4 drop database +sql drop if exists database $db +goto loop_lable \ No newline at end of file diff --git a/tests/script/unique/cluster/cluster_main.sim b/tests/script/unique/cluster/cluster_main.sim index 236f1aa59a3c0aa9091365e8131df4e3ae290afc..4c02d416f3024ed7893bbaa9ae1bef38bab25b4f 100644 --- a/tests/script/unique/cluster/cluster_main.sim +++ b/tests/script/unique/cluster/cluster_main.sim @@ -71,7 +71,7 @@ print ============== step3: create stable stb1 $stb = stb1 sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) -print ============== step4: start 10 client1/ 10 client2/ 10 client3/ 10 client4/ 1 client5 +print ============== step4: start back client1_0.sim run_back unique/cluster/client1_0.sim #run_back unique/cluster/client1_1.sim #run_back unique/cluster/client1_2.sim @@ -106,26 +106,26 @@ endi print wait for a while to let clients start insert data sleep 5000 +print ============== step4-1: add dnode4/dnode5 into cluster +sql create dnode $hostname4 +sql create dnode $hostname5 + $loop_cnt = 0 loop_cluster_do: print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** -print ============== step5: start dnode4/dnode5 and add into cluster, then wait ready +print ============== step5: start dnode4/dnode5 system sh/exec.sh -n dnode4 -s start system sh/exec.sh -n dnode5 -s start -sql create dnode $hostname4 -sql create dnode $hostname5 - -sleep 5000 - +sleep 20000 -print ============== step6: stop and drop dnode1, then remove data dir of dnode1 +print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -sql drop dnode $hostname1 -sleep 5000 +sleep 10000 +#sql drop dnode $hostname1 +#sleep 5000 -system rm -rf ../../../sim/dnode1/data -sleep 20000 +#system rm -rf ../../../sim/dnode1/data +#sleep 20000 sql show mnodes print show mnodes @@ -139,7 +139,22 @@ print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 -return -1 + +print ============== step6-1: restart dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 10000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT @@ -236,14 +251,17 @@ if $data04 != 2 then return -1 endi -print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 +print ============== step14: stop dnode4/dnode5 system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT sleep 20000 -sql drop dnode $hostname4 -sql drop dnode $hostname5 -system rm -rf ../../../sim/dnode4/data -system rm -rf ../../../sim/dnode5/data +#system sh/exec.sh -n dnode4 -s start +#system sh/exec.sh -n dnode5 -s start +#sleep 10000 +#sql drop dnode $hostname4 +#sql drop dnode $hostname5 +#system rm -rf ../../../sim/dnode4/data +#system rm -rf ../../../sim/dnode5/data print ============== step15: alter replica 1 sql alter database $db replica 1 @@ -263,13 +281,13 @@ if $data04 != 2 then return -1 endi -print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready -system sh/cfg.sh -n dnode1 -c first -v $hostname2 -system sh/cfg.sh -n dnode1 -c second -v $hostname3 - -system sh/exec.sh -n dnode1 -s start -sql create dnode $hostname1 -sleep 20000 +#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready +#system sh/cfg.sh -n dnode1 -c first -v $hostname2 +#system sh/cfg.sh -n dnode1 -c second -v $hostname3 +# +#system sh/exec.sh -n dnode1 -s start +#sql create dnode $hostname1 +#sleep 20000 print ============== step18: alter replica 3 sql alter database $db replica 3 diff --git a/tests/script/unique/cluster/cluster_main0.sim b/tests/script/unique/cluster/cluster_main0.sim new file mode 100644 index 0000000000000000000000000000000000000000..f3cce9fd45f90d2d00c509dc1f7f14d2f52278a4 --- /dev/null +++ b/tests/script/unique/cluster/cluster_main0.sim @@ -0,0 +1,288 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 +system sh/cfg.sh -n dnode5 -c walLevel -v 1 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256 + +system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode5 -c alternativeRole -v 0 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1/dnode2/dnode3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 3000 +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 3000 + +print ============== step2: create db1 with replica 3 +$db = db1 +print create database $db replica 3 +#sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica 3 +sql use $db + +print ============== step3: create stable stb1 +$stb = stb1 +print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) +sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) + +print ============== step4: start others client +run_back unique/cluster/client1_0.sim +run_back unique/cluster/client1_1.sim +run_back unique/cluster/client1_2.sim +run_back unique/cluster/client1_3.sim +#run_back unique/cluster/client2_0.sim +#run_back unique/cluster/client2_1.sim +#run_back unique/cluster/client2_2.sim +#run_back unique/cluster/client2_3.sim +#run_back unique/cluster/client3.sim +#run_back unique/cluster/client4.sim + +sleep 20000 + +wait_subsim_insert_complete_create_tables: +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_complete_create_tables +endi + +wait_subsim_insert_data: +print select count(*) from $stb +sql select count(*) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_data +endi + +print wait for a while to let clients start insert data +sleep 5000 + +$loop_cnt = 0 +loop_cluster_do: +print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** +print ============== step5: start dnode4/dnode5 and add into cluster, then wait ready +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start +sql create dnode $hostname4 +sql create dnode $hostname5 + +sleep 5000 + + +print ============== step6: stop dnode1 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 10000 +#sql drop dnode $hostname1 +#sleep 5000 +#system rm -rf ../../../sim/dnode1/data +#sleep 20000 +print ============== step6-1: restart dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 10000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step7: stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step8: restart dnode2, then wait sync end +system sh/exec.sh -n dnode2 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step9: stop dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step10: restart dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step11: stop dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s stop -x SIGINT +sleep 20000 + +print ============== step12: restart dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step13: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +if $data04 != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +sleep 3000 +sql drop dnode $hostname4 +sql drop dnode $hostname5 +system rm -rf ../../../sim/dnode4/data +system rm -rf ../../../sim/dnode5/data +sleep 20000 + +print ============== step15: alter replica 1 +sql alter database $db replica 1 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 1 then + print rplica is not modify to 1, error!!!!!! + return -1 +endi + +print ============== step16: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready +#system sh/cfg.sh -n dnode1 -c first -v $hostname2 +#system sh/cfg.sh -n dnode1 -c second -v $hostname3 +# +#system sh/exec.sh -n dnode1 -s start +#sql create dnode $hostname1 +#sleep 20000 + +print ============== step18: alter replica 3 +sql alter database $db replica 3 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + print rplica is not modify to 3, error!!!!!! + return -1 +endi + +print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** +$loop_cnt = $loop_cnt + 1 +goto loop_cluster_do diff --git a/tests/script/unique/cluster/cluster_main1.sim b/tests/script/unique/cluster/cluster_main1.sim new file mode 100644 index 0000000000000000000000000000000000000000..82e1e2be838893657c77e73caa326141318666a5 --- /dev/null +++ b/tests/script/unique/cluster/cluster_main1.sim @@ -0,0 +1,329 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 +system sh/cfg.sh -n dnode5 -c walLevel -v 1 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256 + +system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode5 -c alternativeRole -v 0 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1/dnode2/dnode3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 3000 +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 3000 + +print ============== step2: create db1 with replica 3 +$replica = 3 +$db = db1 +print create database $db replica $replica +#sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica $replica +sql use $db + +print ============== step3: create stable stb1 +$stb = stb1 +print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) +sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) + +print ============== step4: start others client +run_back unique/cluster/main1_client1_0.sim +run_back unique/cluster/main1_client1_1.sim +run_back unique/cluster/main1_client1_2.sim +run_back unique/cluster/main1_client1_3.sim +run_back unique/cluster/client3.sim +run_back unique/cluster/client4.sim + +sleep 20000 + +wait_subsim_insert_complete_create_tables: +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_complete_create_tables +endi + +wait_subsim_insert_data: +print select count(*) from $stb +sql select count(*) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_data +endi + +print wait for a while to let clients start insert data +sleep 5000 + +print ============== step4-1: add dnode4/dnode5 into cluster +sql create dnode $hostname4 +sql create dnode $hostname5 +sleep 5000 + +$loop_cnt = 0 +loop_cluster_do: +print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** +print ============== step5: start dnode4/dnode5 +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start +sleep 20000 + +print ============== step6: stop dnode1 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 10000 +#sql drop dnode $hostname1 +#sleep 5000 +#system rm -rf ../../../sim/dnode1/data +#sleep 20000 +print ============== step6-1: restart dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 10000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step7: stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step8: restart dnode2, then wait sync end +system sh/exec.sh -n dnode2 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step9: stop dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step10: restart dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step11: stop dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s stop -x SIGINT +sleep 20000 + +#print ============== step12: restart dnode4, then wait sync end +#system sh/exec.sh -n dnode4 -s start +#sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step13: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +print ============== step14: stop dnode5 +system sh/exec.sh -n dnode5 -s stop -x SIGINT +sleep 20000 +#system rm -rf ../../../sim/dnode5/data + +#print ============== step14-1: drop dnode5, then remove data dir +#sql drop dnode $hostname5 +#sleep 20000 +#system rm -rf ../../../sim/dnode5/data + +print ============== step15: alter replica 1 +sql alter database $db replica 1 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 1 then + print rplica is not modify to 1, error!!!!!! + return -1 +endi + +print ============== step16: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready +#system sh/cfg.sh -n dnode1 -c first -v $hostname2 +#system sh/cfg.sh -n dnode1 -c second -v $hostname3 +# +#system sh/exec.sh -n dnode1 -s start +#sql create dnode $hostname1 +#sleep 20000 + +print ============== step18: alter replica 3 +sql alter database $db replica 3 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 3 then + print rplica is not modify to 3, error!!!!!! + return -1 +endi + +print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** +$loop_cnt = $loop_cnt + 1 +goto loop_cluster_do diff --git a/tests/script/unique/cluster/cluster_main2.sim b/tests/script/unique/cluster/cluster_main2.sim new file mode 100644 index 0000000000000000000000000000000000000000..e5fe3f2a217e7eaf5135544818e6210ddd0f33b2 --- /dev/null +++ b/tests/script/unique/cluster/cluster_main2.sim @@ -0,0 +1,333 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 +system sh/cfg.sh -n dnode5 -c walLevel -v 1 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256 + +system sh/cfg.sh -n dnode1 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode2 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode3 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode4 -c alternativeRole -v 0 +system sh/cfg.sh -n dnode5 -c alternativeRole -v 0 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1/dnode2/dnode3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 3000 +sql connect +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 3000 + +print ============== step2: create db1 with replica 3 +$replica = 3 +$db = db1 +print create database $db replica $replica +#sql create database $db replica 3 maxTables $totalTableNum +sql create database $db replica $replica +sql use $db + +print ============== step3: create stable stb1 +$stb = stb1 +print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) +sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16)) + +print ============== step4: start others client +run_back unique/cluster/main2_client1_0.sim +run_back unique/cluster/main2_client1_1.sim +run_back unique/cluster/main2_client1_2.sim +run_back unique/cluster/main2_client1_3.sim +run_back unique/cluster/main2_client2_0.sim +run_back unique/cluster/main2_client2_1.sim +run_back unique/cluster/main2_client2_2.sim +run_back unique/cluster/main2_client2_3.sim +run_back unique/cluster/client3.sim +run_back unique/cluster/client4.sim + +sleep 20000 + +wait_subsim_insert_complete_create_tables: +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_complete_create_tables +endi + +wait_subsim_insert_data: +print select count(*) from $stb +sql select count(*) from $stb +print data00 $data00 +if $data00 < 1000 then + sleep 3000 + goto wait_subsim_insert_data +endi + +print wait for a while to let clients start insert data +sleep 5000 + +print ============== step4-1: add dnode4/dnode5 into cluster +sql create dnode $hostname4 +sql create dnode $hostname5 +sleep 5000 + + +$loop_cnt = 0 +loop_cluster_do: +print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** +print ============== step5: start dnode4/dnode5 +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start + +print ============== step6: stop dnode1 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 10000 +#sql drop dnode $hostname1 +#sleep 5000 +#system rm -rf ../../../sim/dnode1/data +#sleep 20000 +print ============== step6-1: restart dnode1 +system sh/exec.sh -n dnode1 -s start +sleep 10000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step7: stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 + +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step8: restart dnode2, then wait sync end +system sh/exec.sh -n dnode2 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step9: stop dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step10: restart dnode3, then wait sync end +system sh/exec.sh -n dnode3 -s start +sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step11: stop dnode4, then wait sync end +system sh/exec.sh -n dnode4 -s stop -x SIGINT +sleep 20000 + +#print ============== step12: restart dnode4, then wait sync end +#system sh/exec.sh -n dnode4 -s start +#sleep 20000 +sql show mnodes +print show mnodes +print rows: $rows +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 +print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 +print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 +print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 +print $data0_7 $data1_7 $data2_7 $data3_7 $data4_7 +print $data0_8 $data1_8 $data2_8 $data3_8 $data4_8 +print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 + +print ============== step13: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +print ============== step14: drop dnode5, then remove data dir +system sh/exec.sh -n dnode5 -s stop -x SIGINT +sleep 20000 +#system rm -rf ../../../sim/dnode5/data + +#print ============== step14-1: drop dnode5, then remove data dir +#sql drop dnode $hostname5 +#sleep 20000 +#system rm -rf ../../../sim/dnode5/data + +print ============== step15: alter replica 1 +sql alter database $db replica 1 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 1 then + print rplica is not modify to 1, error!!!!!! + return -1 +endi + +print ============== step16: alter replica 2 +sql alter database $db replica 2 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 2 then + print rplica is not modify to 2, error!!!!!! + return -1 +endi + +#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready +#system sh/cfg.sh -n dnode1 -c first -v $hostname2 +#system sh/cfg.sh -n dnode1 -c second -v $hostname3 +# +#system sh/exec.sh -n dnode1 -s start +#sql create dnode $hostname1 +#sleep 20000 + +print ============== step18: alter replica 3 +sql alter database $db replica 3 +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 + +if $data00 == db1 then + $replica = $data04 +elif $data10 == db1 then + $replica = $data14 +else then + print ==== db1 already not exists!!!!! + return -1 +endi + +if $replica != 3 then + print rplica is not modify to 3, error!!!!!! + return -1 +endi + +print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** +$loop_cnt = $loop_cnt + 1 +goto loop_cluster_do diff --git a/tests/script/unique/cluster/main1_client1_0.sim b/tests/script/unique/cluster/main1_client1_0.sim new file mode 100644 index 0000000000000000000000000000000000000000..d4f2aa4294a1e754d531a16c027c966262b1cae6 --- /dev/null +++ b/tests/script/unique/cluster/main1_client1_0.sim @@ -0,0 +1,94 @@ +#system sh/stop_dnodes.sh +#system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 10000 +#system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +#system sh/exec.sh -n dnode1 -s start +#sql connect +#$db = db1 +#sql create database $db +#sql use $db +#$stb = stb1 +#sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) + + +$tblStart = 0 +$tblEnd = 1000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_0 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_0 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop + $x = $x + 20 + $ts = $ts + 40a + goto continue_next_0 + insert_error_loop: + print ============== main1_client1_0 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ... + continue_next_0: + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + if $data00 != $totalRows then + print data00 $data00 totalRows $totalRows + $deltaRows = $totalRows - $data00 + if $lastLossRows == 0 then + $lastLossRows = $deltaRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_0 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + elif $deltaRows != $lastLossRows + $tmp = $deltaRows - $lastLossRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_0 insert loss: $tmp *********** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + $lastLossRows = $deltaRows + endi +# return -1 + endi + goto continue_next_1 + query_error_loop: + print ============== main1_client1_0 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + continue_next_1: + print ====================== client1_0 insert data complete once ............ + endi +endw +print ====================== client1_0 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main1_client1_1.sim b/tests/script/unique/cluster/main1_client1_1.sim new file mode 100644 index 0000000000000000000000000000000000000000..b50f533ce1cede4c1c8fd788bd89c85b64d8d375 --- /dev/null +++ b/tests/script/unique/cluster/main1_client1_1.sim @@ -0,0 +1,82 @@ +$tblStart = 1000 +$tblEnd = 2000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_1 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_1 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop + $x = $x + 20 + $ts = $ts + 40a + goto continue_next_0 + insert_error_loop: + print ============== main1_client1_1 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ... + continue_next_0: + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + if $data00 != $totalRows then + print data00 $data00 totalRows $totalRows + $deltaRows = $totalRows - $data00 + if $lastLossRows == 0 then + $lastLossRows = $deltaRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_1 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + elif $deltaRows != $lastLossRows + $tmp = $deltaRows - $lastLossRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_1 insert loss: $tmp *********** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + $lastLossRows = $deltaRows + endi +# return -1 + endi + goto continue_next_1 + query_error_loop: + print ============== main1_client1_1 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + continue_next_1: + print ====================== client1_2 insert data complete once ............ + endi +endw +print ====================== client1_1 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main1_client1_2.sim b/tests/script/unique/cluster/main1_client1_2.sim new file mode 100644 index 0000000000000000000000000000000000000000..8cc39ded0437379cbd391dad0dfad26cc32a8f05 --- /dev/null +++ b/tests/script/unique/cluster/main1_client1_2.sim @@ -0,0 +1,81 @@ +$tblStart = 2000 +$tblEnd = 3000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_2 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_2 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop + $x = $x + 20 + $ts = $ts + 40a + goto continue_next_0 + insert_error_loop: + print ============== main1_client1_2 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ... + continue_next_0: + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + if $data00 != $totalRows then + print data00 $data00 totalRows $totalRows + $deltaRows = $totalRows - $data00 + if $lastLossRows == 0 then + $lastLossRows = $deltaRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_2 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + elif $deltaRows != $lastLossRows + $tmp = $deltaRows - $lastLossRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_2 insert loss: $tmp *********** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + $lastLossRows = $deltaRows + endi +# return -1 + endi + goto continue_next_1 + query_error_loop: + print ============== main1_client1_2 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + continue_next_1: + print ====================== client1_2 insert data complete once ............ + endi +endw +print ====================== client1_2 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main1_client1_3.sim b/tests/script/unique/cluster/main1_client1_3.sim new file mode 100644 index 0000000000000000000000000000000000000000..72e1be48ac64d34f7c7c06e7ae882de164b3c22c --- /dev/null +++ b/tests/script/unique/cluster/main1_client1_3.sim @@ -0,0 +1,81 @@ +$tblStart = 3000 +$tblEnd = 4000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_3 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_3 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop + $x = $x + 20 + $ts = $ts + 40a + goto continue_next_0 + insert_error_loop: + print ============== main1_client1_3 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ... + continue_next_0: + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + if $data00 != $totalRows then + print data00 $data00 totalRows $totalRows + $deltaRows = $totalRows - $data00 + if $lastLossRows == 0 then + $lastLossRows = $deltaRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_3 insert loss: $deltaRows ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + elif $deltaRows != $lastLossRows + $tmp = $deltaRows - $lastLossRows + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client1_3 insert loss: $tmp *********** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + $lastLossRows = $deltaRows + endi +# return -1 + endi + goto continue_next_1 + query_error_loop: + print ============== main1_client1_3 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop + continue_next_1: + print ====================== client1_3 insert data complete once ............ + endi +endw +print ====================== client1_3 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main1_client4.sim b/tests/script/unique/cluster/main1_client4.sim new file mode 100644 index 0000000000000000000000000000000000000000..eb8049b04beafa2c8a909deae83b26d83d12245d --- /dev/null +++ b/tests/script/unique/cluster/main1_client4.sim @@ -0,0 +1,127 @@ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db2 +$stb = stb2 + +loop_lable: +print ========= start loop create db, table, inset data, alter column/tag, select, drop db + +sql create database if not exists $db +sql use $db +sql create table if not exists $stb ( ts timestamp, c1 int, c2 float ) tags ( t1 int , t2 binary(16) ) +$tagBinary = ' . client4 +$tagBinary = $tagBinary . ' + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +$rowsPerLoop = 100 +$ts = $tsStart + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb -x continue_loop + print data00 $data00 totalRows $totalRows + if $data00 != $totalRows then + print ********************** select error ********************** + endi + continue_loop: + print ====================== client4 insert data complete once ............ + endi +endw + +##################### alter column +sql alter table $stb add column c3 double +sql alter table $stb drop column c2 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb -x continue_loop + print data00 $data00 totalRows $totalRows + if $data00 != $totalRows then + print ********************** select error ********************** + endi + continue_loop: + print ====================== client4 insert data complete once ............ + endi +endw + + +##################### alter tag +sql alter table $stb add tag t3 int +sql alter table $stb drop tag t2 + +$i = $tblStart +while $i < $tblEnd + $tb = ttb . $i + sql create table if not exists $tb using $stb tags ($i, $i) + $i = $i + 1 +endw + +$rowsPerLoop = 100 +$ts = $tsStart + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb -x continue_loop + print data00 $data00 totalRows $totalRows + if $data00 != $totalRows then + print ********************** select error ********************** + endi + continue_loop: + print ====================== client4 insert data complete once ............ + endi +endw + +goto loop_lable \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client1_0.sim b/tests/script/unique/cluster/main2_client1_0.sim new file mode 100644 index 0000000000000000000000000000000000000000..a7fd181363808eb62d36ec8d461ea9ae300a6502 --- /dev/null +++ b/tests/script/unique/cluster/main2_client1_0.sim @@ -0,0 +1,69 @@ +#system sh/stop_dnodes.sh +#system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 10000 +#system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +#system sh/exec.sh -n dnode1 -s start +#sql connect +#$db = db1 +#sql create database $db +#sql use $db +#$stb = stb1 +#sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) + + +$tblStart = 0 +$tblEnd = 1000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_0 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_0 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client1_0 insert data complete once ............ + endi +endw +print ====================== client1_0 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client1_1.sim b/tests/script/unique/cluster/main2_client1_1.sim new file mode 100644 index 0000000000000000000000000000000000000000..f22d62a7411110e3b7b94fd54702dd24c9af1b48 --- /dev/null +++ b/tests/script/unique/cluster/main2_client1_1.sim @@ -0,0 +1,57 @@ +$tblStart = 1000 +$tblEnd = 2000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_1 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_1 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client1_1 insert data complete once ............ + endi +endw +print ====================== client1_1 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client1_2.sim b/tests/script/unique/cluster/main2_client1_2.sim new file mode 100644 index 0000000000000000000000000000000000000000..df3a925c591194b819d8d1fc6bf9234d851fa5b4 --- /dev/null +++ b/tests/script/unique/cluster/main2_client1_2.sim @@ -0,0 +1,56 @@ +$tblStart = 2000 +$tblEnd = 3000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_2 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_2 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client1_2 insert data complete once ............ + endi +endw +print ====================== client1_2 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client1_3.sim b/tests/script/unique/cluster/main2_client1_3.sim new file mode 100644 index 0000000000000000000000000000000000000000..9c22432fa5176ec8cb36f6549e312170803263db --- /dev/null +++ b/tests/script/unique/cluster/main2_client1_3.sim @@ -0,0 +1,56 @@ +$tblStart = 3000 +$tblEnd = 4000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client1_3 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client1_3 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client1_3 insert data complete once ............ + endi +endw +print ====================== client1_3 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client2_0.sim b/tests/script/unique/cluster/main2_client2_0.sim new file mode 100644 index 0000000000000000000000000000000000000000..e44efc49fc7333541d5870ed05eb1cb1a289c3a2 --- /dev/null +++ b/tests/script/unique/cluster/main2_client2_0.sim @@ -0,0 +1,56 @@ +$tblStart = 0 +$tblEnd = 1000 +$tsStart = 1325347200001 # 2012-01-01 00:00:00.001 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client2_0 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client2_0 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lastLossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client2_0 insert data complete once ............ + endi +endw +print ====================== client2_0 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client2_1.sim b/tests/script/unique/cluster/main2_client2_1.sim new file mode 100644 index 0000000000000000000000000000000000000000..19de147d3ea79b54702b8ce0da30d2bff69841f2 --- /dev/null +++ b/tests/script/unique/cluster/main2_client2_1.sim @@ -0,0 +1,57 @@ +$tblStart = 1000 +$tblEnd = 2000 +$tsStart = 1325347200001 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client2_1 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client2_1 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client2_1 insert data complete once ............ + endi +endw +print ====================== client2_1 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client2_2.sim b/tests/script/unique/cluster/main2_client2_2.sim new file mode 100644 index 0000000000000000000000000000000000000000..19e3540bd1676ebb15d0d318488feea61ef9074a --- /dev/null +++ b/tests/script/unique/cluster/main2_client2_2.sim @@ -0,0 +1,57 @@ +$tblStart = 2000 +$tblEnd = 3000 +$tsStart = 1325347200001 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client2_2 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client2_2 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client2_2 insert data complete once ............ + endi +endw +print ====================== client2_2 success and auto end ===================== \ No newline at end of file diff --git a/tests/script/unique/cluster/main2_client2_3.sim b/tests/script/unique/cluster/main2_client2_3.sim new file mode 100644 index 0000000000000000000000000000000000000000..219df131c8f31babc25f2204dbddab7f0232c56b --- /dev/null +++ b/tests/script/unique/cluster/main2_client2_3.sim @@ -0,0 +1,57 @@ +$tblStart = 3000 +$tblEnd = 4000 +$tsStart = 1325347200001 # 2012-01-01 00:00:00.000 +############################################################### + +sql connect + +$db = db1 +$stb = stb1 + +sql use $db + + +######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8)) +$tagBinary = ' . client2_3 +$tagBinary = $tagBinary . ' +#print ======= tag: $tagBinary + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb using $stb tags ( $i , $tagBinary ) + sql create table if not exists $tb using $stb tags ( $i , $tagBinary ) + $i = $i + 1 +endw + +print ====================== client2_3 create table end, start insert data ............ +sql select count(tbname) from $stb +print select count(tbname) from $stb +print data00 $data00 + +$rowsPerLoop = 100 +$ts = $tsStart + +$lossRows = 0 + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) + $x = $x + 20 + $ts = $ts + 40a + endw + + $totalRows = $totalRows + $x + $i = $i + 1 + + if $i == $tblEnd then + $i = $tblStart + + sql select count(*) from $stb where t2 == $tagBinary + print ====================== client2_3 insert data complete once ............ + endi +endw +print ====================== client2_3 success and auto end ===================== \ No newline at end of file