diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 719d80aa775bf3b5abe3065ef7eca799d452e315..12ea4ad78d18a5f2b09daabb71093c153d30aca7 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -318,7 +318,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->maxPoints = maxRows; pCols->bufSize = maxRowSize * maxRows; - pCols->buf = calloc(1, pCols->bufSize); + pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { free(pCols); return NULL; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 305302b71a4716c8f5aed2dca0f2df7ac78d7c98..79cfbe2a37f2236d54d49a37150745c4d75016ec 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -361,7 +361,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -954,13 +954,13 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "index = %d", i); + sprintf(tempS, "areaid = %d", i); } else { - sprintf(tempS, " or index = %d ", i); + sprintf(tempS, " or areaid = %d ", i); } strcat(condition, tempS); - sprintf(command, "select %s from m1 where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 0ae86007568d3d7a00ba6f7b147ac5f0d00ee74a..1856223391ae719ef98492160da9810826b983a9 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) +ENDIF() diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2325d12d92dc4b862a1041ab8fc952a5a1ac9630..4790c22668d669d300994b59bef748d9bb80fbd7 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -446,7 +446,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set the idle timer to monitor the activity taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); - pConn->secured = 1; // connection shall be secured + + // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) + pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 210d95853cf216e93dd09feb4f8e03e15b269471..762d2253e21aca6c95318cdbcda16a31da9b45cf 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -42,6 +42,7 @@ extern int tsdbDebugFlag; #define TSDB_MAX_TABLE_SCHEMAS 16 #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF // Definitions // ------------------ tsdbMeta.c @@ -132,21 +133,30 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { +#ifdef TSDB_IDX + TSDB_FILE_TYPE_IDX = 0, + TSDB_FILE_TYPE_HEAD, +#else TSDB_FILE_TYPE_HEAD = 0, +#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, +#ifdef TSDB_IDX + TSDB_FILE_TYPE_NIDX, +#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; typedef struct { - uint32_t offset; + uint32_t magic; uint32_t len; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size uint32_t totalBlocks; uint32_t totalSubBlocks; + uint32_t offset; + uint64_t size; // total size of the file + uint64_t tombSize; // unused file size } STsdbFileInfo; typedef struct { @@ -197,6 +207,7 @@ typedef struct { // ------------------ tsdbRWHelper.c typedef struct { + int32_t tid; uint32_t len; uint32_t offset; uint32_t hasLast : 2; @@ -220,7 +231,7 @@ typedef struct { typedef struct { int32_t delimiter; // For recovery usage - int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int32_t tid; uint64_t uid; SCompBlock blocks[]; } SCompInfo; @@ -249,24 +260,27 @@ typedef struct { typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { - int fid; - TSKEY minKey; - TSKEY maxKey; - // For read/write purpose - SFile headF; - SFile dataF; - SFile lastF; - // For write purpose only - SFile nHeadF; - SFile nLastF; + TSKEY minKey; + TSKEY maxKey; + SFileGroup fGroup; +#ifdef TSDB_IDX + SFile nIdxF; +#endif + SFile nHeadF; + SFile nLastF; } SHelperFile; typedef struct { uint64_t uid; int32_t tid; - int32_t sversion; } SHelperTable; +typedef struct { + SCompIdx* pIdxArray; + int numOfIdx; + int curIdx; +} SIdxH; + typedef struct { tsdb_rw_helper_t type; @@ -274,7 +288,9 @@ typedef struct { int8_t state; // For file set usage SHelperFile files; - SCompIdx* pCompIdx; + SIdxH idxH; + SCompIdx curCompIdx; + void* pWIdx; // For table set usage SHelperTable tableInfo; SCompInfo* pCompInfo; @@ -286,7 +302,6 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; - // Operations // ------------------ tsdbMeta.c #define TABLE_TYPE(t) (t)->type @@ -296,6 +311,7 @@ typedef struct { #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid #define TABLE_LASTKEY(t) (t)->lastKey +#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); @@ -445,6 +461,16 @@ void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS #define helperRepo(h) (h)->pRepo #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) +#define helperFileId(h) ((h)->files.fGroup.fileId) +#ifdef TSDB_IDX +#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#endif +#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) +#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) +#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) +#define helperNewHeadF(h) (&((h)->files.nHeadF)) +#define helperNewLastF(h) (&((h)->files.nLastF)) int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 95cc47292b1c24735fb564c7a6091fdeba801868..767fbc82529f401be1c25186e67af23c773cf5c7 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,11 @@ #include "ttime.h" #include "tfile.h" +#ifdef TSDB_IDX +const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; +#else const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +112,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +130,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +143,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +160,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +173,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -260,6 +264,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { } pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; if (tsdbUpdateFileHeader(pFile, 0) < 0) { tsdbCloseFile(pFile); @@ -323,23 +328,25 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; - tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); - tlen += taosEncodeFixedU64(buf, pInfo->size); - tlen += taosEncodeFixedU64(buf, pInfo->tombSize); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); return tlen; } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); return buf; } @@ -358,7 +365,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e30164592dee23c1baf86c11f44daba2f3e4ab85..772bcf48d6023f9d0070ba8c23ce8eab128ddd4e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -212,59 +212,61 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *sdup = strdup(pRepo->rootDir); char *prefix = dirname(sdup); + int prefixLen = strlen(prefix); + tfree(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { fname = tsdbGetMetaFileName(pRepo->rootDir); *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { - tfree(sdup); return 0; } } else { SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - fname = strdup(pFGroup->files[(*index) % 3].fname); + fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); + magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; } else { - if (pFGroup->fileId * 3 + 2 < eindex) { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { fname = strdup(pFGroup->files[0].fname); - *index = pFGroup->fileId * 3; + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFGroup->files[0].info.magic; } else { - tfree(sdup); return 0; } } } - strcpy(name, fname + strlen(prefix)); + strcpy(name, fname + prefixLen); } else { // get the named file at the specified index. If not there, return 0 if (*index == TSDB_META_FILE_INDEX) { // get meta file fname = tsdbGetMetaFileName(pRepo->rootDir); + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found - tfree(sdup); return 0; } - SFile *pFile = &pFGroup->files[(*index) % 3]; + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; fname = strdup(pFile->fname); + magic = pFile->info.magic; } } if (stat(fname, &fState) < 0) { - tfree(sdup); tfree(fname); return 0; } - tfree(sdup); *size = fState.st_size; - magic = *size; + // magic = *size; tfree(fname); return magic; @@ -793,7 +795,8 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - SCompIdx *pIdx = &rhelper.pCompIdx[i]; + tsdbSetHelperTable(&rhelper, pTable, pRepo); + SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 1b7db635b337e2576ac61ac100dabad91be793df..5325eb5b0cc24b11884191645601b9d3c4c5d2eb 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -627,9 +627,12 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); - pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; - pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; - pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; +#ifdef TSDB_IDX + pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); +#endif + pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); pthread_rwlock_unlock(&(pFileH->fhlock)); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0d52b7ae3348f588871424f97a5c1476b150963c..f8792469cbcd4699046856d4f41bdaf0e789a0dc 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -99,6 +99,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); + SFile *pFile = NULL; // Clear the helper object tsdbResetHelper(pHelper); @@ -106,44 +107,51 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); // Set the files - pHelper->files.fid = pGroup->fileId; - pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; - pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; - pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; + pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, pHelper->files.nHeadF.fname); - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, pHelper->files.nLastF.fname); +#ifdef TSDB_IDX + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); +#endif + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files - if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; +#ifdef TSDB_IDX + if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; +#endif + if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; + +#ifdef TSDB_IDX + // Create and open .i file + pFile = helperNewIdxF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; +#endif // Create and open .h - if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; - // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + pFile = helperNewHeadF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { - if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + pFile = helperNewLastF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) goto _err; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; } } else { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) goto _err; } helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); @@ -155,59 +163,98 @@ _err: } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { - if (pHelper->files.headF.fd > 0) { - close(pHelper->files.headF.fd); - pHelper->files.headF.fd = -1; + SFile *pFile = NULL; + +#ifdef TSDB_IDX + pFile = helperIdxF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; + } +#endif + + pFile = helperHeadF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.dataF.fd > 0) { + + pFile = helperDataF(pHelper); + if (pFile->fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); - fsync(pHelper->files.dataF.fd); + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); } - close(pHelper->files.dataF.fd); - pHelper->files.dataF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.lastF.fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - fsync(pHelper->files.lastF.fd); + + pFile = helperLastF(pHelper); + if (pFile->fd > 0) { + if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { + fsync(pFile->fd); } - close(pHelper->files.lastF.fd); - pHelper->files.lastF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } + if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (pHelper->files.nHeadF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); - fsync(pHelper->files.nHeadF.fd); - close(pHelper->files.nHeadF.fd); - pHelper->files.nHeadF.fd = -1; +#ifdef TSDB_IDX + pFile = helperNewIdxF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nHeadF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nHeadF.fname, - pHelper->files.headF.fname, strerror(errno)); + if (rename(pFile->fname, helperIdxF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperIdxF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.headF.info = pHelper->files.nHeadF.info; + helperIdxF(pHelper)->info = pFile->info; + } + } +#endif + + pFile = helperNewHeadF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; + if (hasError) { + (void)remove(pFile->fname); + } else { + if (rename(pFile->fname, helperHeadF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperHeadF(pHelper)->fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + helperHeadF(pHelper)->info = pFile->info; } } - if (pHelper->files.nLastF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); - fsync(pHelper->files.nLastF.fd); - close(pHelper->files.nLastF.fd); - pHelper->files.nLastF.fd = -1; + pFile = helperNewLastF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nLastF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nLastF.fname, - pHelper->files.lastF.fname, strerror(errno)); + if (rename(pFile->fname, helperLastF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperLastF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.lastF.info = pHelper->files.nLastF.info; + helperLastF(pHelper)->info = helperNewLastF(pHelper)->info; } } } @@ -224,18 +271,35 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { pHelper->tableInfo.tid = pTable->tableId.tid; pHelper->tableInfo.uid = pTable->tableId.uid; STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - pHelper->tableInfo.sversion = schemaVersion(pSchema); tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - SCompIdx *pIdx = pHelper->pCompIdx + pTable->tableId.tid; - if (pIdx->offset > 0) { - if (pIdx->uid != TABLE_UID(pTable)) { - memset((void *)pIdx, 0, sizeof(SCompIdx)); - } else { - if (pIdx->hasLast) pHelper->hasOldLastBlock = true; + if (pHelper->idxH.numOfIdx > 0) { + while (true) { + if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + break; + } + + SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); + if (pIdx->tid == TABLE_TID(pTable)) { + if (pIdx->uid == TABLE_UID(pTable)) { + pHelper->curCompIdx = *pIdx; + } else { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } + pHelper->idxH.curIdx++; + break; + } else if (pIdx->tid > TABLE_TID(pTable)) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + break; + } else { + pHelper->idxH.curIdx++; + } } + } else { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } helperSetState(pHelper, TSDB_HELPER_TABLE_SET); @@ -245,8 +309,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = &(pHelper->pCompIdx[TABLE_TID(pCommitIter->pTable)]); - int blkIdx = 0; + SCompIdx *pIdx = &(pHelper->curCompIdx); + int blkIdx = 0; ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable)); if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -271,44 +335,53 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { STsdbCfg *pCfg = &pHelper->pRepo->config; ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last); + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && + pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) + return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + +#if 0 if (pCompBlock->numOfSubBlocks > 1) { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], &compBlock, true, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) return -1; if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; } else { - if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.lastF.fname, + if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pCompBlock->offset = lseek(pHelper->files.nLastF.fd, 0, SEEK_END); + pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END); if (pCompBlock->offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nLastF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, pCompBlock->len) < pCompBlock->len) { + if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), - pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); + helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } +#endif pHelper->hasOldLastBlock = false; } @@ -317,164 +390,178 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { } int tsdbWriteCompInfo(SRWHelper *pHelper) { - off_t offset = 0; - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (pIdx->offset > 0) { - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pIdx->offset = offset; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + SCompIdx *pIdx = &(pHelper->curCompIdx); + off_t offset = 0; + SFile * pFile = helperNewHeadF(pHelper); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - } else { - if (pIdx->len > 0) { + if (pIdx->len > 0) { + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { + if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; + } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; - pHelper->pCompInfo->checksum = 0; + pHelper->pCompInfo->tid = pHelper->tableInfo.tid; ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + } - if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.nHeadF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pIdx->offset = offset; + pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; + ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + +#ifdef TSDB_IDX + pFile = helperNewIdxF(pHelper); +#endif + + if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } } + + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); + pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; } int tsdbWriteCompIdx(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + off_t offset = 0; + +#ifdef TSDB_IDX + SFile *pFile = helperNewIdxF(pHelper); +#else + SFile *pFile = helperNewHeadF(pHelper); +#endif + + pFile->info.len += sizeof(TSCKSUM); + if (tsizeof(pHelper->pWIdx) < pFile->info.len) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, pFile->info.len); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); + + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s to end since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, - strerror(errno)); + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - SFile *pFile = &(pHelper->files.nHeadF); pFile->info.offset = offset; - void *buf = pHelper->pBuffer; - for (uint32_t i = 0; i < pCfg->maxTables; i++) { - SCompIdx *pCompIdx = pHelper->pCompIdx + i; - if (pCompIdx->offset > 0) { - int drift = POINTER_DISTANCE(buf, pHelper->pBuffer); - if (tsizeof(pHelper->pBuffer) - drift < 128) { - pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - buf = POINTER_SHIFT(pHelper->pBuffer, drift); - taosEncodeVariantU32(&buf, i); - tsdbEncodeSCompIdx(&buf, pCompIdx); - } - } - - int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pBuffer, tsize) < tsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, - pHelper->files.nHeadF.fname, strerror(errno)); + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pFile->info.len = tsize; + return 0; } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); +#ifdef TSDB_IDX + SFile *pFile = helperIdxF(pHelper); +#else + SFile *pFile = helperHeadF(pHelper); +#endif + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - SFile *pFile = &(pHelper->files.headF); - int fd = pFile->fd; - - memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); - if (pFile->info.offset > 0) { - ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); + if (pFile->info.len > 0) { + if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s to %u since %s", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, strerror(errno)); + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { - tsdbError("vgId:%d file %s SCompIdx part is corrupted. offset %u len %u", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, pFile->info.len); + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } // Decode it + pHelper->idxH.numOfIdx = 0; void *ptr = pHelper->pBuffer; while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { - uint32_t tid = 0; - if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; - ASSERT(tid > 0 && tid < pCfg->maxTables); + size_t tlen = tsizeof(pHelper->idxH.pIdxArray); + pHelper->idxH.numOfIdx++; - if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; + if (tlen < pHelper->idxH.numOfIdx * sizeof(SCompIdx)) { + pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); + if (pHelper->idxH.pIdxArray == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } - ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); - } + ptr = tsdbDecodeSCompIdx(ptr, &(pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1])); + if (ptr == NULL) { + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + ASSERT(pHelper->idxH.numOfIdx == 1 || pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1].tid > + pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 2].tid); + + ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + pFile->info.len = 0; + } + // Copy the memory for outside usage - if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); + if (target && pHelper->idxH.numOfIdx > 0) + memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); return 0; } @@ -482,15 +569,15 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); - int fd = pHelper->files.headF.fd; + int fd = helperHeadF(pHelper)->fd; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { ASSERT(pIdx->uid == pHelper->tableInfo.uid); if (lseek(fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.headF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -499,18 +586,18 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, strerror(errno)); + helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) { tsdbError("vgId:%d file %s SCompInfo part is corrupted, tid %d uid %" PRIu64, REPO_ID(pHelper->pRepo), - pHelper->files.headF.fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); + helperHeadF(pHelper)->fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } - ASSERT(pIdx->uid == pHelper->pCompInfo->uid); + ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); } helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); @@ -523,7 +610,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); if (lseek(pFile->fd, pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); @@ -642,9 +729,9 @@ _err: // ---------------------- INTERNAL FUNCTIONS ---------------------- static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - ASSERT(pHelper->files.lastF.fd > 0); + ASSERT(helperLastF(pHelper)->fd > 0); struct stat st; - if (fstat(pHelper->files.lastF.fd, &st) < 0) return true; + if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true; if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; return false; } @@ -729,6 +816,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa ASSERT(flen > 0); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + pFile->info.magic = + taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); if (ncol != 0) { pCompCol->offset = toffset; @@ -747,6 +836,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompData->numOfCols = nColsNotAllNull; taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); + pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)), + sizeof(TSCKSUM)); // Write the whole block to file if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) { @@ -804,7 +895,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { } static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks); ASSERT(pCompBlock->numOfSubBlocks == 1); @@ -851,7 +942,7 @@ _err: static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { ASSERT(pCompBlock->numOfSubBlocks == 0); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -935,7 +1026,7 @@ _err: static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { ASSERT(pCompBlock->numOfSubBlocks == 1); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); @@ -971,24 +1062,21 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { + pHelper->idxH.numOfIdx = 0; + pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - pHelper->files.fid = -1; - pHelper->files.headF.fd = -1; - pHelper->files.dataF.fd = -1; - pHelper->files.lastF.fd = -1; - pHelper->files.nHeadF.fd = -1; - pHelper->files.nLastF.fd = -1; + helperHeadF(pHelper)->fd = -1; + helperDataF(pHelper)->fd = -1; + helperLastF(pHelper)->fd = -1; + helperNewHeadF(pHelper)->fd = -1; + helperNewLastF(pHelper)->fd = -1; +#ifdef TSDB_IDX + helperIdxF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; +#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); - if (pHelper->pCompIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbResetHelperFileImpl(pHelper); return 0; } @@ -996,7 +1084,8 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); tsdbResetHelperFileImpl(pHelper); - tzfree(pHelper->pCompIdx); + tzfree(pHelper->idxH.pIdxArray); + tzfree(pHelper->pWIdx); } // ---------- Operations on Helper Table part @@ -1154,7 +1243,7 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, ASSERT(pCompBlock->numOfSubBlocks <= 1); ASSERT(colIds[0] == 0); - SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile * pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); SCompCol compCol = {0}; // If only load timestamp column, no need to load SCompData part @@ -1215,7 +1304,7 @@ _err: static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len); if (pHelper->pBuffer == NULL) { @@ -1314,6 +1403,7 @@ _err: static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { int tlen = 0; + tlen += taosEncodeVariantI32(buf, pIdx->tid); tlen += taosEncodeVariantU32(buf, pIdx->len); tlen += taosEncodeVariantU32(buf, pIdx->offset); tlen += taosEncodeFixedU8(buf, pIdx->hasLast); @@ -1329,6 +1419,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { uint32_t numOfBlocks = 0; uint64_t value = 0; + if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; @@ -1346,7 +1437,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; SCompBlock compBlock = {0}; @@ -1362,7 +1453,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; @@ -1393,7 +1484,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int *blkIdx) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; @@ -1427,7 +1518,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; } else { @@ -1466,7 +1557,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; tblkIdx++; } @@ -1493,7 +1584,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, false) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1506,7 +1597,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; @@ -1577,10 +1668,10 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, ASSERT(pDataCols->numOfRows > 0); if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pFile = &(pHelper->files.dataF); + pFile = helperDataF(pHelper); } else { isLast = true; - pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); + pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper); } ASSERT(pFile->fd > 0); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7538c6f7e175e49b339ffd3741de4a54e06d6421..3b6e90b740003aa03297dbe35c214e995b1f7608 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -126,12 +126,13 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); +static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle); +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle); +static int tsdbCheckInfoCompar(const void* key1, const void* key2); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -236,7 +237,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } - + + taosArraySort(pQueryHandle->pTableCheckInfo, tsdbCheckInfoCompar); pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); @@ -554,7 +556,9 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); pCheckInfo->numOfBlocks = 0; - SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; + tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); + + SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx; // no data block in this file, try next file if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) { @@ -571,8 +575,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo pCheckInfo->compSize = compIndex->len; } - tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); - tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); SCompInfo* pCompInfo = pCheckInfo->pCompInfo; @@ -2461,3 +2463,13 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} \ No newline at end of file diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 346e567c41e0a209b7dde96de55faa74a704f9b3..6d67607e24113771e09d2f8707ae5930fb21b20a 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -25,10 +25,11 @@ typedef int (*iterFunc)(void *, void *cont, int contLen); typedef void (*afterFunc)(void *); typedef struct { - int64_t size; // including 512 bytes of header size - int64_t tombSize; - int64_t nRecords; - int64_t nDels; + int64_t size; // including 512 bytes of header size + int64_t tombSize; + int64_t nRecords; + int64_t nDels; + uint32_t magic; } SStoreInfo; typedef struct { @@ -45,6 +46,8 @@ typedef struct { SStoreInfo info; } SKVStore; +#define KVSTORE_MAGIC(s) (s)->info.magic + int tdCreateKVStore(char *fname); int tdDestroyKVStore(char *fname); SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index ab2aa738c66b68684883a0f02a59bba4a5b90667..0704285da5fd1cc390c11891e2983c32ed5f5265 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -34,6 +34,7 @@ #define TD_KVSTORE_MAINOR_VERSION 0 #define TD_KVSTORE_SNAP_SUFFIX ".snap" #define TD_KVSTORE_NEW_SUFFIX ".new" +#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF typedef struct { uint64_t uid; @@ -140,6 +141,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; pStore->info.size = TD_KVSTORE_HEADER_SIZE; + pStore->info.magic = info.magic; if (tdRestoreKVStore(pStore) < 0) goto _err; @@ -251,6 +253,8 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe return -1; } + pStore->info.magic = + taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); pStore->info.size += (sizeof(SKVRecord) + contLen); SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { // just to insert @@ -288,6 +292,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { return -1; } + pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, POINTER_DISTANCE(pBuf, buf)); pStore->info.size += POINTER_DISTANCE(pBuf, buf); pStore->info.nDels++; pStore->info.nRecords--; @@ -371,7 +376,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { } static int tdInitKVStoreHeader(int fd, char *fname) { - SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0}; + SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC}; return tdUpdateKVStoreHeader(fd, fname, &info); } @@ -382,6 +387,7 @@ static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { tlen += taosEncodeVariantI64(buf, pInfo->tombSize); tlen += taosEncodeVariantI64(buf, pInfo->nRecords); tlen += taosEncodeVariantI64(buf, pInfo->nDels); + tlen += taosEncodeFixedU32(buf, pInfo->magic); return tlen; } @@ -391,6 +397,7 @@ static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); return buf; } diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 9f66eba37a3eccf68b7f0b584eb9a28fc0dfcd1d..b0b5d3013b250e109f0c5a0ef4fc2c23c4563406 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread gcov) +ENDIF() diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index a96115dfb256aed446da05bb8a1e2fd46b8a8437..73c4b952148ff0700840049ab7cb3867d2c9eece 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -29,7 +29,7 @@ if __name__ == '__main__': # Create a database named db try: - c1.execute('create database db') + c1.execute('create database if not exists db ') except Exception as err: conn.close() raise(err) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9d1aef0dc5cc2cf266931ff6e8ba88794649686c..904e712fa46a9f409d8196759569b22c84a091b6 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -144,6 +144,7 @@ python3 ./test.py -f query/querySort.py python3 ./test.py -f query/queryJoin.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/metric_1.py @@ -161,3 +162,22 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py \ No newline at end of file diff --git a/tests/pytest/functions/function_avg.py b/tests/pytest/functions/function_avg.py new file mode 100644 index 0000000000000000000000000000000000000000..9481550ba3f9d912b734d19b7c794c495fffdecd --- /dev/null +++ b/tests/pytest/functions/function_avg.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # average verifacation + tdSql.error("select avg(ts) from test") + tdSql.error("select avg(ts) from test1") + tdSql.error("select avg(col7) from test") + tdSql.error("select avg(col7) from test1") + tdSql.error("select avg(col8) from test") + tdSql.error("select avg(col8) from test1") + tdSql.error("select avg(col9) from test") + tdSql.error("select avg(col9) from test1") + + tdSql.query("select avg(col1) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col2) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col3) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col4) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col5) from test") + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col6) from test") + tdSql.checkData(0, 0, np.average(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py new file mode 100644 index 0000000000000000000000000000000000000000..4074166f927acbbcbb5899a692ff87f3c187f92f --- /dev/null +++ b/tests/pytest/functions/function_bottom.py @@ -0,0 +1,93 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + # bottom verifacation + tdSql.error("select bottom(ts, 10) from test") + tdSql.error("select bottom(col1, 0) from test") + tdSql.error("select bottom(col1, 101) from test") + tdSql.error("select bottom(col2, 0) from test") + tdSql.error("select bottom(col2, 101) from test") + tdSql.error("select bottom(col3, 0) from test") + tdSql.error("select bottom(col3, 101) from test") + tdSql.error("select bottom(col4, 0) from test") + tdSql.error("select bottom(col4, 101) from test") + tdSql.error("select bottom(col5, 0) from test") + tdSql.error("select bottom(col5, 101) from test") + tdSql.error("select bottom(col6, 0) from test") + tdSql.error("select bottom(col6, 101) from test") + tdSql.error("select bottom(col7, 10) from test") + tdSql.error("select bottom(col8, 10) from test") + tdSql.error("select bottom(col9, 10) from test") + + tdSql.query("select bottom(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + tdSql.query("select bottom(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py new file mode 100644 index 0000000000000000000000000000000000000000..4795a6b2dfe72bb9fa747b550be9f56c0c255d88 --- /dev/null +++ b/tests/pytest/functions/function_count.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + # Count verifacation + tdSql.query("select count(*) from test") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(ts) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col1) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col2) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col3) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col4) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col5) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col6) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col7) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col8) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col9) from test") + tdSql.checkData(0, 0, 10) + + tdSql.execute("alter table test add column col10 int") + tdSql.query("select count(col10) from test") + tdSql.checkRows(0) + + tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)") + tdSql.query("select count(col10) from test") + tdSql.checkData(0, 0, 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py new file mode 100644 index 0000000000000000000000000000000000000000..b6f496d7d68ebaa0f19dfc947d7745802198a3dc --- /dev/null +++ b/tests/pytest/functions/function_diff.py @@ -0,0 +1,99 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + + # diff verifacation + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + tdSql.error("select diff(ts) from test") + tdSql.error("select diff(ts) from test1") + tdSql.error("select diff(col1) from test") + tdSql.error("select diff(col2) from test") + tdSql.error("select diff(col3) from test") + tdSql.error("select diff(col4) from test") + tdSql.error("select diff(col5) from test") + tdSql.error("select diff(col6) from test") + tdSql.error("select diff(col7) from test") + tdSql.error("select diff(col7) from test1") + tdSql.error("select diff(col8) from test") + tdSql.error("select diff(col8) from test1") + tdSql.error("select diff(col9) from test") + tdSql.error("select diff(col9) from test1") + + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py new file mode 100644 index 0000000000000000000000000000000000000000..2b78bd33d8bdb0bcd6775215b1cf0b1e398f2928 --- /dev/null +++ b/tests/pytest/functions/function_first.py @@ -0,0 +1,119 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, False) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata1') + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last.py b/tests/pytest/functions/function_last.py new file mode 100644 index 0000000000000000000000000000000000000000..337c18a28c2cd99d556e5b017377edc56671c490 --- /dev/null +++ b/tests/pytest/functions/function_last.py @@ -0,0 +1,119 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # last verifacation + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last_row.py b/tests/pytest/functions/function_last_row.py new file mode 100644 index 0000000000000000000000000000000000000000..ea5cf661eb9e70e2ed06ebd712b2f65dfa62beeb --- /dev/null +++ b/tests/pytest/functions/function_last_row.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # last_row verifacation + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_leastsquares.py b/tests/pytest/functions/function_leastsquares.py new file mode 100644 index 0000000000000000000000000000000000000000..0ef0a2b78612dca604f7a7a7f70841c917376a17 --- /dev/null +++ b/tests/pytest/functions/function_leastsquares.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + # leastsquares verifacation + tdSql.error("select leastsquares(ts, 1, 1) from test1") + tdSql.error("select leastsquares(col1, 1, 1) from test") + tdSql.error("select leastsquares(col2, 1, 1) from test") + tdSql.error("select leastsquares(col3, 1, 1) from test") + tdSql.error("select leastsquares(col4, 1, 1) from test") + tdSql.error("select leastsquares(col5, 1, 1) from test") + tdSql.error("select leastsquares(col6, 1, 1) from test") + tdSql.error("select leastsquares(col7, 1, 1) from test1") + tdSql.error("select leastsquares(col8, 1, 1) from test1") + tdSql.error("select leastsquares(col9, 1, 1) from test1") + + tdSql.query("select leastsquares(col1, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col2, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col3, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col4, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col5, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + tdSql.query("select leastsquares(col6, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py new file mode 100644 index 0000000000000000000000000000000000000000..3bd5031276adfd76574a1a43e311a1d6ec81ab16 --- /dev/null +++ b/tests/pytest/functions/function_max.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # max verifacation + tdSql.error("select max(ts) from test") + tdSql.error("select max(ts) from test1") + tdSql.error("select max(col7) from test") + tdSql.error("select max(col7) from test1") + tdSql.error("select max(col8) from test") + tdSql.error("select max(col8) from test1") + tdSql.error("select max(col9) from test") + tdSql.error("select max(col9) from test1") + + tdSql.query("select max(col1) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col2) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col3) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col4) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col5) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + tdSql.query("select max(col6) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py new file mode 100644 index 0000000000000000000000000000000000000000..bc180bc22414a4a3aa3d233438b10a04fd08bc2b --- /dev/null +++ b/tests/pytest/functions/function_min.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select min(ts) from test") + tdSql.error("select min(ts) from test1") + tdSql.error("select min(col7) from test") + tdSql.error("select min(col7) from test1") + tdSql.error("select min(col8) from test") + tdSql.error("select min(col8) from test1") + tdSql.error("select min(col9) from test") + tdSql.error("select min(col9) from test1") + + tdSql.query("select min(col1) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col2) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col3) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col4) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col5) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + tdSql.query("select min(col6) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py new file mode 100644 index 0000000000000000000000000000000000000000..36810621cb2adac494f2c3332c13f31b8ff69962 --- /dev/null +++ b/tests/pytest/functions/function_operations.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + # min verifacation + tdSql.error("select ts + col1 from test") + tdSql.error("select ts + col1 from test1") + tdSql.error("select col1 + col7 from test") + tdSql.error("select col1 + col7 from test1") + tdSql.error("select col1 + col8 from test") + tdSql.error("select col1 + col8 from test1") + tdSql.error("select col1 + col9 from test") + tdSql.error("select col1 + col9 from test1") + + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 2.0) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(10) + tdSql.checkData(1, 0, 6.0) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 3.2) + + tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py new file mode 100644 index 0000000000000000000000000000000000000000..aaeb94372e44282da0d83849c18e40857463b42c --- /dev/null +++ b/tests/pytest/functions/function_percentile.py @@ -0,0 +1,140 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20))''') + for i in range(self.rowNum): + tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # percentile verifacation + tdSql.error("select percentile(ts 20) from test") + tdSql.error("select apercentile(ts 20) from test") + tdSql.error("select percentile(col7 20) from test") + tdSql.error("select apercentile(col7 20) from test") + tdSql.error("select percentile(col8 20) from test") + tdSql.error("select apercentile(col8 20) from test") + tdSql.error("select percentile(col9 20) from test") + tdSql.error("select apercentile(col9 20) from test") + + tdSql.query("select percentile(col1, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col1, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col1, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col1, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col2, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col2, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col2, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col2, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col3, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col3, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col3, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col3, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col4, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col4, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col4, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col4, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col5, 0) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 0)) + tdSql.query("select apercentile(col5, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 50) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 50)) + tdSql.query("select apercentile(col5, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 100) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 100)) + tdSql.query("select apercentile(col5, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col6, 0) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 0)) + tdSql.query("select apercentile(col6, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 50) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 50)) + tdSql.query("select apercentile(col6, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 100) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 100)) + tdSql.query("select apercentile(col6, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_spread.py b/tests/pytest/functions/function_spread.py new file mode 100644 index 0000000000000000000000000000000000000000..462933597388090f069f9319751aaea032ea4133 --- /dev/null +++ b/tests/pytest/functions/function_spread.py @@ -0,0 +1,106 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + + # spread verifacation + tdSql.query("select spread(ts) from test1") + tdSql.checkRows(1) + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + + tdSql.error("select spread(col7) from test") + tdSql.error("select spread(col7) from test1") + tdSql.error("select spread(col8) from test") + tdSql.error("select spread(col8) from test1") + tdSql.error("select spread(col9) from test") + tdSql.error("select spread(col9) from test1") + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py new file mode 100644 index 0000000000000000000000000000000000000000..23df415aa35feecf84ba40a50604d2b86fc7f008 --- /dev/null +++ b/tests/pytest/functions/function_stddev.py @@ -0,0 +1,80 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # stddev verifacation + tdSql.error("select stddev(ts) from test1") + tdSql.error("select stddev(col1) from test") + tdSql.error("select stddev(col2) from test") + tdSql.error("select stddev(col3) from test") + tdSql.error("select stddev(col4) from test") + tdSql.error("select stddev(col5) from test") + tdSql.error("select stddev(col6) from test") + tdSql.error("select stddev(col7) from test1") + tdSql.error("select stddev(col8) from test1") + tdSql.error("select stddev(col9) from test1") + + tdSql.query("select stddev(col1) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col2) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col3) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col4) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col5) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + tdSql.query("select stddev(col6) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_sum.py b/tests/pytest/functions/function_sum.py new file mode 100644 index 0000000000000000000000000000000000000000..ef5ddfe3158cc3c3072fb474bdc42d50a8de30f0 --- /dev/null +++ b/tests/pytest/functions/function_sum.py @@ -0,0 +1,69 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # sum verifacation + tdSql.error("select sum(ts) from test") + tdSql.error("select sum(col7) from test") + tdSql.error("select sum(col8) from test") + tdSql.error("select sum(col9) from test") + + tdSql.query("select sum(col1) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col2) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col3) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col4) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col5) from test") + tdSql.checkData(0, 0, np.sum(floatData)) + tdSql.query("select sum(col6) from test") + tdSql.checkData(0, 0, np.sum(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py new file mode 100644 index 0000000000000000000000000000000000000000..e24ff1cc5339c79e1a87653e9b528ec34f3c2ea9 --- /dev/null +++ b/tests/pytest/functions/function_top.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # top verifacation + tdSql.error("select top(ts, 10) from test") + tdSql.error("select top(col1, 0) from test") + tdSql.error("select top(col1, 101) from test") + tdSql.error("select top(col2, 0) from test") + tdSql.error("select top(col2, 101) from test") + tdSql.error("select top(col3, 0) from test") + tdSql.error("select top(col3, 101) from test") + tdSql.error("select top(col4, 0) from test") + tdSql.error("select top(col4, 101) from test") + tdSql.error("select top(col5, 0) from test") + tdSql.error("select top(col5, 101) from test") + tdSql.error("select top(col6, 0) from test") + tdSql.error("select top(col6, 101) from test") + tdSql.error("select top(col7, 10) from test") + tdSql.error("select top(col8, 10) from test") + tdSql.error("select top(col9, 10) from test") + + tdSql.query("select top(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + tdSql.query("select top(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_twa.py b/tests/pytest/functions/function_twa.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce4c99b6041e2cfd6bccdbc0e00e5fd8037f9d9 --- /dev/null +++ b/tests/pytest/functions/function_twa.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # twa verifacation + tdSql.error("select twa(ts) from test") + tdSql.error("select twa(ts) from test1") + + tdSql.error("select twa(col1) from test") + tdSql.error("select twa(col1) from test1") + + tdSql.error("select twa(col2) from test") + tdSql.error("select twa(col2) from test1") + + tdSql.error("select twa(col3) from test") + tdSql.error("select twa(col3) from test1") + + tdSql.error("select twa(col4) from test") + tdSql.error("select twa(col4) from test1") + + tdSql.error("select twa(col5) from test") + tdSql.error("select twa(col5) from test1") + + tdSql.error("select twa(col6) from test") + tdSql.error("select twa(col6) from test1") + + tdSql.error("select twa(col7) from test") + tdSql.error("select twa(col7) from test1") + + tdSql.error("select twa(col8) from test") + tdSql.error("select twa(col8) from test1") + + tdSql.error("select twa(col9) from test") + tdSql.error("select twa(col9) from test1") + + tdSql.error("select twa(col1) from test where ts > %d" % self.ts) + tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col2) from test where ts > %d" % self.ts) + tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col3) from test where ts > %d" % self.ts) + tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col4) from test where ts > %d" % self.ts) + tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col5) from test where ts > %d" % self.ts) + tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col6) from test where ts > %d" % self.ts) + tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 1db5497604923cecd8111015dd0ba497b914b453..5033ffdb48237e6a758d67162af5757cce04252d 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -231,10 +231,12 @@ class TDTestCase: tdSql.error("select * from st where tagcol1 like '____'") # > for nchar type on tag - tdSql.error("select * from st where tagcol2 > 'table'") + tdSql.query("select * from st where tagcol2 > 'table1'") + tdSql.checkRows(5) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 >= 'table'") + tdSql.query("select * from st where tagcol2 >= 'table1'") + tdSql.checkRows(10) # = for nchar type on tag tdSql.query("select * from st where tagcol2 = 'table1'") @@ -249,10 +251,12 @@ class TDTestCase: tdSql.checkRows(10) # > for nchar type on tag - tdSql.error("select * from st where tagcol2 < 'table'") + tdSql.query("select * from st where tagcol2 < 'table'") + tdSql.checkRows(0) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 <= 'table'") + tdSql.query("select * from st where tagcol2 <= 'table'") + tdSql.checkRows(0) # % for nchar type on tag case 1 tdSql.query("select * from st where tagcol2 like '%'") @@ -291,10 +295,12 @@ class TDTestCase: tdSql.checkRows(10) # > for binary type on tag - tdSql.error("select * from st where tagcol3 > '表'") + tdSql.query("select * from st where tagcol3 > '表'") + tdSql.checkRows(10) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 >= '表'") + tdSql.query("select * from st where tagcol3 >= '表'") + tdSql.checkRows(10) # = for binary type on tag tdSql.query("select * from st where tagcol3 = '水表'") @@ -309,10 +315,12 @@ class TDTestCase: tdSql.checkRows(5) # > for binary type on tag - tdSql.error("select * from st where tagcol3 < '水表'") + tdSql.query("select * from st where tagcol3 < '水表'") + tdSql.checkRows(0) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 <= '水表'") + tdSql.query("select * from st where tagcol3 <= '水表'") + tdSql.checkRows(5) # % for binary type on tag case 1 tdSql.query("select * from st where tagcol3 like '%'") diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 24cd93f0fc58772e2d1741d9ee0119d9d5247357..61ec491f5de6ed946aafe3e0f549c233f6a250b7 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/stream1.py @@ -155,3 +156,23 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index ec39ab61b9be2a52031dfcbb786cd6bc88fef555..1e1d02959f58e716a534a164aa72e3d18176d7c0 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -122,11 +122,16 @@ class TDSql: return self.cursor.istype(col, dataType) def checkData(self, row, col, data): - self.checkRowCol(row, col) - if self.queryResult[row][col] != data: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) - tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + self.checkRowCol(row, col) + if self.queryResult[row][col] != data: + if isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (self.sql, row, col, self.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) if data is None: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % @@ -135,9 +140,12 @@ class TDSql: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + elif isinstance(data, float): tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - else: + else: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 2f45ab077c095a89d04b780052f5c36ac67983ee..15cc2954e87c7c6f2e23472e8e22d347844bc6cf 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -135,7 +135,6 @@ run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim #unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim #unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim @@ -212,12 +211,9 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim run general/stream/table_1.sim -run general/stream/metrics_1.sim run general/stream/table_n.sim run general/stream/metrics_n.sim run general/stream/table_del.sim diff --git a/tests/script/sh/exec-no-random-fail.sh b/tests/script/sh/exec-no-random-fail.sh index 04a663bc5aed52208df7183013f6989945e5f760..2bd0a64923f73df010e40f427d40cb9e212480d7 100755 --- a/tests/script/sh/exec-no-random-fail.sh +++ b/tests/script/sh/exec-no-random-fail.sh @@ -88,7 +88,9 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 0 > /dev/null 2>&1 & fi @@ -99,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh index a354021684884781e0157b011cf378dfb00dd924..3761498859591188799d1ad3814ed3b01372269d 100755 --- a/tests/script/sh/exec-random-fail.sh +++ b/tests/script/sh/exec-random-fail.sh @@ -88,9 +88,12 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else - nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & + nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail \ + --random-file-fail-factor 5 > /dev/null 2>&1 & fi else @@ -99,12 +102,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 2f294075a1695298cb52b53bc52e11d9dce9fe07..1c84a6b10ed3f958e096d32eda727441ff2591da 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -101,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`