Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
未验证 提交 9f96559a 编写于 作者: H Hongze Cheng 提交者： GitHub
浏览文件
操作

Merge pull request #6531 from taosdata/feature/TD-4352 

[TD-4352]compact tsdb meta data feature
上级 f2627a91 23cbcdd0
显示空白变更内容
内联 并排
Showing with 1674 addition and 49 deletion
packaging/cfg/taos.cfg
浏览文件 @ 9f96559a
...@@ -284,3 +284,5 @@ keepColumnName 1 ...@@ -284,3 +284,5 @@ keepColumnName 1
# 0 no query allowed, queries are disabled # 0 no query allowed, queries are disabled
# queryBufferSize -1 # queryBufferSize -1
# percent of redundant data in tsdb meta will compact meta data,0 means donot compact
# tsdbMetaCompactRatio 0
src/common/src/tglobal.c
浏览文件 @ 9f96559a
...@@ -150,6 +150,7 @@ int32_t tsMaxVgroupsPerDb = 0; ...@@ -150,6 +150,7 @@ int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO;
// tsdb config // tsdb config
...@@ -1581,6 +1582,16 @@ static void doInitGlobalConfig(void) { ...@@ -1581,6 +1582,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "tsdbMetaCompactRatio";
cfg.ptr = &tsTsdbMetaCompactRatio;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 100;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
assert(tsGlobalConfigNum <= TSDB_CFG_MAX_NUM); assert(tsGlobalConfigNum <= TSDB_CFG_MAX_NUM);
#ifdef TD_TSZ #ifdef TD_TSZ
// lossy compress // lossy compress
......
src/inc/taosdef.h
浏览文件 @ 9f96559a
...@@ -277,6 +277,7 @@ do { \ ...@@ -277,6 +277,7 @@ do { \
#define TSDB_MAX_TABLES 10000000 #define TSDB_MAX_TABLES 10000000
#define TSDB_DEFAULT_TABLES 1000000 #define TSDB_DEFAULT_TABLES 1000000
#define TSDB_TABLES_STEP 1000 #define TSDB_TABLES_STEP 1000
#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default
#define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650 #define TSDB_MAX_DAYS_PER_FILE 3650
......
src/tsdb/inc/tsdbFS.h
浏览文件 @ 9f96559a
...@@ -44,6 +44,7 @@ typedef struct { ...@@ -44,6 +44,7 @@ typedef struct {
SFSStatus* cstatus; // current status SFSStatus* cstatus; // current status
SHashObj* metaCache; // meta cache SHashObj* metaCache; // meta cache
SHashObj* metaCacheComp; // meta cache for compact
bool intxn; bool intxn;
SFSStatus* nstatus; // new status SFSStatus* nstatus; // new status
} STsdbFS; } STsdbFS;
......
src/tsdb/src/tsdbCommit.c
浏览文件 @ 9f96559a
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
*/ */
#include "tsdbint.h" #include "tsdbint.h"
extern int32_t tsTsdbMetaCompactRatio;
#define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_SUBBLOCKS 8
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
if (key < 0) { if (key < 0) {
...@@ -55,8 +57,9 @@ typedef struct { ...@@ -55,8 +57,9 @@ typedef struct {
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
static int tsdbCommitMeta(STsdbRepo *pRepo); static int tsdbCommitMeta(STsdbRepo *pRepo);
static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact);
static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid);
static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile);
static int tsdbCommitTSData(STsdbRepo *pRepo); static int tsdbCommitTSData(STsdbRepo *pRepo);
static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbStartCommit(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo, int eno); static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
...@@ -261,6 +264,35 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { ...@@ -261,6 +264,35 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
// =================== Commit Meta Data // =================== Commit Meta Data
static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) {
STsdbFS * pfs = REPO_FS(pRepo);
SMFile * pOMFile = pfs->cstatus->pmf;
SDiskID did;
// Create/Open a meta file or open the existing file
if (pOMFile == NULL) {
// Create a new meta file
did.level = TFS_PRIMARY_LEVEL;
did.id = TFS_PRIMARY_ID;
tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
if (open && tsdbCreateMFile(pMf, true) < 0) {
tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf));
} else {
tsdbInitMFileEx(pMf, pOMFile);
if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) {
tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
}
return 0;
}
static int tsdbCommitMeta(STsdbRepo *pRepo) { static int tsdbCommitMeta(STsdbRepo *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo); STsdbFS * pfs = REPO_FS(pRepo);
SMemTable *pMem = pRepo->imem; SMemTable *pMem = pRepo->imem;
...@@ -269,43 +301,34 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { ...@@ -269,43 +301,34 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
SActObj * pAct = NULL; SActObj * pAct = NULL;
SActCont * pCont = NULL; SActCont * pCont = NULL;
SListNode *pNode = NULL; SListNode *pNode = NULL;
SDiskID did;
ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0);
if (listNEles(pMem->actList) <= 0) { if (listNEles(pMem->actList) <= 0) {
// no meta data to commit, just keep the old meta file // no meta data to commit, just keep the old meta file
tsdbUpdateMFile(pfs, pOMFile); tsdbUpdateMFile(pfs, pOMFile);
return 0; if (tsTsdbMetaCompactRatio > 0) {
} else { if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) {
// Create/Open a meta file or open the existing file
if (pOMFile == NULL) {
// Create a new meta file
did.level = TFS_PRIMARY_LEVEL;
did.id = TFS_PRIMARY_ID;
tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateMFile(&mf, true) < 0) {
tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1; return -1;
} }
int ret = tsdbCompactMetaFile(pRepo, pfs, &mf);
if (ret < 0) tsdbError("compact meta file error");
tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); return ret;
}
return 0;
} else { } else {
tsdbInitMFileEx(&mf, pOMFile); if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) {
if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1; return -1;
} }
} }
}
// Loop to write // Loop to write
while ((pNode = tdListPopHead(pMem->actList)) != NULL) { while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
pAct = (SActObj *)pNode->data; pAct = (SActObj *)pNode->data;
if (pAct->act == TSDB_UPDATE_META) { if (pAct->act == TSDB_UPDATE_META) {
pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) {
tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno)); tstrerror(terrno));
tsdbCloseMFile(&mf); tsdbCloseMFile(&mf);
...@@ -338,6 +361,10 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { ...@@ -338,6 +361,10 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
tsdbCloseMFile(&mf); tsdbCloseMFile(&mf);
tsdbUpdateMFile(pfs, &mf); tsdbUpdateMFile(pfs, &mf);
if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) {
tsdbError("compact meta file error");
}
return 0; return 0;
} }
...@@ -375,7 +402,7 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { ...@@ -375,7 +402,7 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) {
pRtn->minFid, pRtn->midFid, pRtn->maxFid); pRtn->minFid, pRtn->midFid, pRtn->maxFid);
} }
static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) {
char buf[64] = "\0"; char buf[64] = "\0";
void * pBuf = buf; void * pBuf = buf;
SKVRecord rInfo; SKVRecord rInfo;
...@@ -401,13 +428,18 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void ...@@ -401,13 +428,18 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void
} }
tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid));
SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache;
pMFile->info.nRecords++;
SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid));
if (pRecord != NULL) { if (pRecord != NULL) {
pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
} else { } else {
pMFile->info.nRecords++; pMFile->info.nRecords++;
} }
taosHashPut(pfs->metaCache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
return 0; return 0;
} }
...@@ -442,6 +474,129 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { ...@@ -442,6 +474,129 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
return 0; return 0;
} }
static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) {
float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords);
float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size);
float compactRatio = (float)(tsTsdbMetaCompactRatio)/100;
if (delPercent < compactRatio && tombPercent < compactRatio) {
return 0;
}
if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
tsdbError("open meta file %s compact fail", pMFile->f.rname);
return -1;
}
tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64,
tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size);
SMFile mf;
SDiskID did;
// first create tmp meta file
did.level = TFS_PRIMARY_LEVEL;
did.id = TFS_PRIMARY_ID;
tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1);
if (tsdbCreateMFile(&mf, true) < 0) {
tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
// second iterator metaCache
int code = -1;
int64_t maxBufSize = 1024;
SKVRecord *pRecord;
void *pBuf = NULL;
pBuf = malloc((size_t)maxBufSize);
if (pBuf == NULL) {
goto _err;
}
// init Comp
assert(pfs->metaCacheComp == NULL);
pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pfs->metaCacheComp == NULL) {
goto _err;
}
pRecord = taosHashIterate(pfs->metaCache, NULL);
while (pRecord) {
if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
tstrerror(terrno));
goto _err;
}
if (pRecord->size > maxBufSize) {
maxBufSize = pRecord->size;
void* tmp = realloc(pBuf, (size_t)maxBufSize);
if (tmp == NULL) {
goto _err;
}
pBuf = tmp;
}
int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
if (nread < 0) {
tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
tstrerror(terrno));
goto _err;
}
if (nread < pRecord->size) {
tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
goto _err;
}
if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) {
tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid,
tstrerror(terrno));
goto _err;
}
pRecord = taosHashIterate(pfs->metaCache, pRecord);
}
code = 0;
_err:
if (code == 0) TSDB_FILE_FSYNC(&mf);
tsdbCloseMFile(&mf);
tsdbCloseMFile(pMFile);
if (code == 0) {
// rename meta.tmp -> meta
tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile));
taosRename(mf.f.aname,pMFile->f.aname);
tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN);
tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN);
// update current meta file info
pfs->nstatus->pmf = NULL;
tsdbUpdateMFile(pfs, &mf);
taosHashCleanup(pfs->metaCache);
pfs->metaCache = pfs->metaCacheComp;
pfs->metaCacheComp = NULL;
} else {
// remove meta.tmp file
remove(mf.f.aname);
taosHashCleanup(pfs->metaCacheComp);
pfs->metaCacheComp = NULL;
}
tfree(pBuf);
ASSERT(mf.info.nDels == 0);
ASSERT(mf.info.tombSize == 0);
tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64,
code,mf.info.nRecords,mf.info.size);
return code;
}
// =================== Commit Time-Series Data // =================== Commit Time-Series Data
static int tsdbCommitTSData(STsdbRepo *pRepo) { static int tsdbCommitTSData(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem; SMemTable *pMem = pRepo->imem;
......
src/tsdb/src/tsdbFS.c
浏览文件 @ 9f96559a
...@@ -215,6 +215,7 @@ STsdbFS *tsdbNewFS(STsdbCfg *pCfg) { ...@@ -215,6 +215,7 @@ STsdbFS *tsdbNewFS(STsdbCfg *pCfg) {
} }
pfs->intxn = false; pfs->intxn = false;
pfs->metaCacheComp = NULL;
pfs->nstatus = tsdbNewFSStatus(maxFSet); pfs->nstatus = tsdbNewFSStatus(maxFSet);
if (pfs->nstatus == NULL) { if (pfs->nstatus == NULL) {
......
src/tsdb/src/tsdbFile.c
浏览文件 @ 9f96559a
...@@ -20,7 +20,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { ...@@ -20,7 +20,7 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"data", // TSDB_FILE_DATA "data", // TSDB_FILE_DATA
"last", // TSDB_FILE_LAST "last", // TSDB_FILE_LAST
"", // TSDB_FILE_MAX "", // TSDB_FILE_MAX
"meta" // TSDB_FILE_META "meta", // TSDB_FILE_META
}; };
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
......
src/tsdb/src/tsdbMeta.c
浏览文件 @ 9f96559a
...@@ -43,6 +43,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); ...@@ -43,6 +43,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable);
static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable);
static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid);
static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema);
static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable);
static int tsdbAddSchema(STable *pTable, STSchema *pSchema); static int tsdbAddSchema(STable *pTable, STSchema *pSchema);
static void tsdbFreeTableSchema(STable *pTable); static void tsdbFreeTableSchema(STable *pTable);
...@@ -128,21 +129,16 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { ...@@ -128,21 +129,16 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
tsdbUnlockRepoMeta(pRepo); tsdbUnlockRepoMeta(pRepo);
// Write to memtable action // Write to memtable action
// TODO: refactor duplicate codes
int tlen = 0;
void *pBuf = NULL;
if (newSuper || superChanged) { if (newSuper || superChanged) {
tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, super); // add insert new super table action
pBuf = tsdbAllocBytes(pRepo, tlen); if (tsdbInsertNewTableAction(pRepo, super) != 0) {
if (pBuf == NULL) goto _err; goto _err;
void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, super); }
ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); }
// add insert new table action
if (tsdbInsertNewTableAction(pRepo, table) != 0) {
goto _err;
} }
tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table);
pBuf = tsdbAllocBytes(pRepo, tlen);
if (pBuf == NULL) goto _err;
void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, table);
ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen);
if (tsdbCheckCommit(pRepo) < 0) return -1; if (tsdbCheckCommit(pRepo) < 0) return -1;
...@@ -383,7 +379,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { ...@@ -383,7 +379,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
tdDestroyTSchemaBuilder(&schemaBuilder); tdDestroyTSchemaBuilder(&schemaBuilder);
} }
// Chage in memory // Change in memory
if (pNewSchema != NULL) { // change super table tag schema if (pNewSchema != NULL) { // change super table tag schema
TSDB_WLOCK_TABLE(pTable->pSuper); TSDB_WLOCK_TABLE(pTable->pSuper);
STSchema *pOldSchema = pTable->pSuper->tagSchema; STSchema *pOldSchema = pTable->pSuper->tagSchema;
...@@ -426,6 +422,21 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { ...@@ -426,6 +422,21 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
} }
// ------------------ INTERNAL FUNCTIONS ------------------ // ------------------ INTERNAL FUNCTIONS ------------------
static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable) {
int tlen = 0;
void *pBuf = NULL;
tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable);
pBuf = tsdbAllocBytes(pRepo, tlen);
if (pBuf == NULL) {
return -1;
}
void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, pTable);
ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen);
return 0;
}
STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) {
STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta));
if (pMeta == NULL) { if (pMeta == NULL) {
...@@ -617,6 +628,7 @@ int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { ...@@ -617,6 +628,7 @@ int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) {
if (pTable->lastCols == NULL) { if (pTable->lastCols == NULL) {
return -1; return -1;
} }
// TODO: use binary search instead
for (int16_t i = 0; i < pTable->maxColNum; ++i) { for (int16_t i = 0; i < pTable->maxColNum; ++i) {
if (pTable->lastCols[i].colId == colId) { if (pTable->lastCols[i].colId == colId) {
return i; return i;
...@@ -734,10 +746,10 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, ...@@ -734,10 +746,10 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema,
TSDB_WUNLOCK_TABLE(pCTable); TSDB_WUNLOCK_TABLE(pCTable);
if (insertAct) { if (insertAct) {
int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); if (tsdbInsertNewTableAction(pRepo, pCTable) != 0) {
void *buf = tsdbAllocBytes(pRepo, tlen); tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " tsdbInsertNewTableAction fail", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
ASSERT(buf != NULL); TABLE_TID(pTable), TABLE_UID(pTable));
tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pCTable); }
} }
} }
......
tests/pytest/manualTest/TD-5114/continueCreateDn.py 0 → 100644
浏览文件 @ 9f96559a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import sys
sys.path.insert(0, os.getcwd())
from util.log import *
from util.sql import *
from util.dnodes import *
import taos
import threading
import subprocess
from random import choice
class TwoClients:
def initConnection(self):
self.host = "chr03"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos/"
self.port =6030
self.rowNum = 10
self.ts = 1537146000000
def run(self):
# new taos client
conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn1)
cur1 = conn1.cursor()
tdSql.init(cur1, True)
tdSql.execute("drop database if exists db3")
# insert data with taosc
for i in range(10):
os.system("taosdemo -f manualTest/TD-5114/insertDataDb3Replica2.json -y ")
# # check data correct
tdSql.execute("show databases")
tdSql.execute("use db3")
tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 20000)
tdSql.query("select count (*) from stb0")
tdSql.checkData(0, 0, 4000000)
# insert data with python connector , if you want to use this case ,cancel note.
# for x in range(10):
# dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"]
# tdSql.execute("drop database if exists db3")
# tdSql.execute("create database db3 keep 3650 replica 2 ")
# tdSql.execute("use db3")
# tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
# col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(3000), tag1 int)''')
# rowNum2= 988
# for i in range(rowNum2):
# tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) )
# rowNum3= 988
# for i in range(rowNum3):
# tdSql.execute("alter table test drop column col%d ;" %( i+14) )
# self.rowNum = 50
# self.rowNum2 = 2000
# self.ts = 1537146000000
# for j in range(self.rowNum2):
# tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
# for i in range(self.rowNum):
# tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
# % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
# # check data correct
# tdSql.execute("show databases")
# tdSql.execute("use db3")
# tdSql.query("select count (tbname) from test")
# tdSql.checkData(0, 0, 200)
# tdSql.query("select count (*) from test")
# tdSql.checkData(0, 0, 200000)
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf manualTest/TD-5114/%s.sql" % testcaseFilename )
clients = TwoClients()
clients.initConnection()
# clients.getBuildPath()
clients.run()
\ No newline at end of file
tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db3",
"drop": "yes",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 20000,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 1000,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 2000,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/manualTest/TD-5114/rollingUpgrade.py 0 → 100644
浏览文件 @ 9f96559a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from sys import version
from fabric import Connection
import random
import time
import datetime
import logging
import subprocess
import os
import sys
class Node:
def __init__(self, index, username, hostIP, password, version):
self.index = index
self.username = username
self.hostIP = hostIP
# self.hostName = hostName
# self.homeDir = homeDir
self.version = version
self.verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % self.version
self.installPath = "TDengine-enterprise-server-%s" % self.version
# self.corePath = '/coredump'
self.conn = Connection("{}@{}".format(username, hostIP), connect_kwargs={"password": "{}".format(password)})
def buildTaosd(self):
try:
print(self.conn)
# self.conn.run('echo "1234" > /home/chr/installtest/test.log')
self.conn.run("cd /home/chr/installtest/ && tar -xvf %s " %self.verName)
self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
except Exception as e:
print("Build Taosd error for node %d " % self.index)
logging.exception(e)
pass
def rebuildTaosd(self):
try:
print(self.conn)
# self.conn.run('echo "1234" > /home/chr/installtest/test.log')
self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
except Exception as e:
print("Build Taosd error for node %d " % self.index)
logging.exception(e)
pass
def startTaosd(self):
try:
self.conn.run("sudo systemctl start taosd")
except Exception as e:
print("Start Taosd error for node %d " % self.index)
logging.exception(e)
def restartTarbi(self):
try:
self.conn.run("sudo systemctl restart tarbitratord ")
except Exception as e:
print("Start Taosd error for node %d " % self.index)
logging.exception(e)
def clearData(self):
timeNow = datetime.datetime.now()
# timeYes = datetime.datetime.now() + datetime.timedelta(days=-1)
timStr = timeNow.strftime('%Y%m%d%H%M%S')
# timStr = timeNow.strftime('%Y%m%d%H%M%S')
try:
# self.conn.run("mv /var/lib/taos/ /var/lib/taos%s " % timStr)
self.conn.run("rm -rf /home/chr/data/taos*")
except Exception as e:
print("rm -rf /var/lib/taos error %d " % self.index)
logging.exception(e)
def stopTaosd(self):
try:
self.conn.run("sudo systemctl stop taosd")
except Exception as e:
print("Stop Taosd error for node %d " % self.index)
logging.exception(e)
def restartTaosd(self):
try:
self.conn.run("sudo systemctl restart taosd")
except Exception as e:
print("Stop Taosd error for node %d " % self.index)
logging.exception(e)
class oneNode:
def FirestStartNode(self, id, username, IP, passwd, version):
# get installPackage
verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version
# installPath = "TDengine-enterprise-server-%s" % self.version
node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0')
node131.conn.run('sshpass -p tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id))
node131.conn.close()
# install TDengine at 192.168.103/104/141
try:
node = Node(id, username, IP, passwd, version)
node.conn.run('echo "start taosd"')
node.buildTaosd()
# clear DataPath , if need clear data
node.clearData()
node.startTaosd()
if id == 103 :
node.restartTarbi()
print("start taosd ver:%s node:%d successfully " % (version,id))
node.conn.close()
# query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
# assert query_pid == 1 , "node %d: start taosd failed " % id
except Exception as e:
print("Stop Taosd error for node %d " % id)
logging.exception(e)
def startNode(self, id, username, IP, passwd, version):
# start TDengine
try:
node = Node(id, username, IP, passwd, version)
node.conn.run('echo "restart taosd"')
# clear DataPath , if need clear data
node.clearData()
node.restartTaosd()
time.sleep(5)
if id == 103 :
node.restartTarbi()
print("start taosd ver:%s node:%d successfully " % (version,id))
node.conn.close()
# query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
# assert query_pid == 1 , "node %d: start taosd failed " % id
except Exception as e:
print("Stop Taosd error for node %d " % id)
logging.exception(e)
def firstUpgradeNode(self, id, username, IP, passwd, version):
# get installPackage
verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version
# installPath = "TDengine-enterprise-server-%s" % self.version
node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0')
node131.conn.run('echo "upgrade cluster"')
node131.conn.run('sshpass -p tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id))
node131.conn.close()
# upgrade TDengine at 192.168.103/104/141
try:
node = Node(id, username, IP, passwd, version)
node.conn.run('echo "start taosd"')
node.conn.run('echo "1234" > /home/chr/test.log')
node.buildTaosd()
time.sleep(5)
node.startTaosd()
if id == 103 :
node.restartTarbi()
print("start taosd ver:%s node:%d successfully " % (version,id))
node.conn.close()
# query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
# assert query_pid == 1 , "node %d: start taosd failed " % id
except Exception as e:
print("Stop Taosd error for node %d " % id)
logging.exception(e)
def upgradeNode(self, id, username, IP, passwd, version):
# backCluster TDengine at 192.168.103/104/141
try:
node = Node(id, username, IP, passwd, version)
node.conn.run('echo "rollback taos"')
node.rebuildTaosd()
time.sleep(5)
node.startTaosd()
if id == 103 :
node.restartTarbi()
print("start taosd ver:%s node:%d successfully " % (version,id))
node.conn.close()
except Exception as e:
print("Stop Taosd error for node %d " % id)
logging.exception(e)
# how to use : cd TDinternal/commumity/test/pytest && python3 manualTest/rollingUpgrade.py ,when inserting data, we can start " python3 manualTest/rollingUpagrade.py". add example "oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')"
# node103=oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
# node104=oneNode().FirestStartNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
# node141=oneNode().FirestStartNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
# node103=oneNode().startNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
# time.sleep(30)
# node141=oneNode().startNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
# time.sleep(30)
# node104=oneNode().startNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
# time.sleep(30)
# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5')
# time.sleep(30)
# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5')
# time.sleep(30)
# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5')
# time.sleep(30)
# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
# time.sleep(30)
# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
# time.sleep(30)
# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
# time.sleep(30)
# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
# time.sleep(30)
# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
# time.sleep(30)
# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')
# time.sleep(30)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
# time.sleep(120)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
# time.sleep(180)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
# time.sleep(240)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5')
# time.sleep(120)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5')
# time.sleep(120)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5')
# time.sleep(180)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
# time.sleep(120)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
# time.sleep(120)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
# time.sleep(180)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
# time.sleep(180)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
# time.sleep(180)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')
# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.9')
# time.sleep(5)
# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.9')
# time.sleep(5)
# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.9')
# time.sleep(30)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
# time.sleep(12)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
# time.sleep(12)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
# time.sleep(180)
# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
# time.sleep(120)
# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
# time.sleep(120)
# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')
tests/pytest/tsdb/insertDataDb1.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 10,
"num_of_records_per_req": 1000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db1",
"drop": "yes",
"replica": 1,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1000,
"childtable_prefix": "stb00_",
"auto_create_table": "no",
"batch_create_tbl_num": 1000,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 1000,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 10000,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 1000,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 200,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/insertDataDb1Replica2.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 10,
"num_of_records_per_req": 1000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db1",
"drop": "yes",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1000,
"childtable_prefix": "stb00_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 100,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 1000,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 200,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/insertDataDb2.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "yes",
"replica": 1,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 200000,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 1000,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 0,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 2,
"childtable_prefix": "stb1_",
"auto_create_table": "no",
"batch_create_tbl_num": 1000,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 5,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/insertDataDb2Newstab.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "no",
"replica": 1,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 0,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"yes",
"childtable_count": 1,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 10,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-11-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/insertDataDb2NewstabReplica2.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "no",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 1,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 0,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"yes",
"childtable_count": 1,
"childtable_prefix": "stb01_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 10,
"childtable_limit": -1,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-11-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/insertDataDb2Replica2.json 0 → 100644
浏览文件 @ 9f96559a
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 4,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 3000,
"max_sql_len": 1024000,
"databases": [{
"dbinfo": {
"name": "db2",
"drop": "yes",
"replica": 2,
"days": 10,
"cache": 50,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"cachelast":0,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb0",
"child_table_exists":"no",
"childtable_count": 2000,
"childtable_prefix": "stb0_",
"auto_create_table": "no",
"batch_create_tbl_num": 100,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 2000,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
},
{
"name": "stb1",
"child_table_exists":"no",
"childtable_count": 2,
"childtable_prefix": "stb1_",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"insert_rows": 5,
"childtable_limit": 0,
"childtable_offset":0,
"interlace_rows": 0,
"insert_interval":0,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
}]
}]
}
tests/pytest/tsdb/tsdbComp.py 0 → 100644
浏览文件 @ 9f96559a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from distutils.log import debug
import sys
import os
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import subprocess
from random import choice
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def getBuildPath(self):
global selfPath
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def run(self):
# set path para
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath+ "/build/bin/"
testPath = selfPath+ "/../../../"
walFilePath = testPath + "/sim/dnode1/data/mnode_bak/wal/"
#new db and insert data
tdSql.execute("drop database if exists db2")
os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath)
tdSql.execute("drop database if exists db1")
os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath)
tdSql.execute("drop table if exists db2.stb0")
os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath)
tdSql.execute("use db2")
tdSql.execute("drop table if exists stb1_0")
tdSql.execute("drop table if exists stb1_1")
tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
tdSql.execute("alter table db2.stb0 add column c4 int")
tdSql.execute("alter table db2.stb0 drop column c2")
tdSql.execute("alter table db2.stb0 add tag t3 int;")
tdSql.execute("alter table db2.stb0 drop tag t1")
tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ")
tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
tdSql.execute("alter table stb2_0 add column c2 binary(4)")
tdSql.execute("alter table stb2_0 drop column c1")
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
# create db utest
dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"]
tdSql.execute("drop database if exists utest")
tdSql.execute("create database utest keep 3650")
tdSql.execute("use utest")
tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
# rowNum1 = 13
# for i in range(rowNum1):
# columnName= "col" + str(i+1)
# tdSql.execute("alter table test drop column %s ;" % columnName )
rowNum2= 988
for i in range(rowNum2):
tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) )
rowNum3= 988
for i in range(rowNum3):
tdSql.execute("alter table test drop column col%d ;" %( i+14) )
self.rowNum = 1
self.rowNum2 = 100
self.rowNum3 = 20
self.ts = 1537146000000
for j in range(self.rowNum2):
tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
for i in range(self.rowNum):
tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
% (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
for j in range(self.rowNum2):
tdSql.execute("drop table if exists test%d" % (j+1))
# stop taosd and restart taosd
tdDnodes.stop(1)
sleep(10)
tdDnodes.start(1)
sleep(5)
tdSql.execute("reset query cache")
query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
print(query_pid2)
# verify that the data is correct
tdSql.execute("use db2")
tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 1)
tdSql.query("select count (tbname) from stb1")
tdSql.checkRows(0)
tdSql.query("select count(*) from stb0_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2)
tdSql.execute("use utest")
tdSql.query("select count (tbname) from test")
tdSql.checkData(0, 0, 1)
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf tsdb/%s.sql" % testcaseFilename )
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
tests/pytest/tsdb/tsdbCompCluster.py 0 → 100644
浏览文件 @ 9f96559a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import sys
sys.path.insert(0, os.getcwd())
from util.log import *
from util.sql import *
from util.dnodes import *
import taos
import threading
import subprocess
from random import choice
class TwoClients:
def initConnection(self):
self.host = "chenhaoran02"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos/"
self.port =6030
self.rowNum = 10
self.ts = 1537146000000
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def run(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath+ "/build/bin/"
walFilePath = "/var/lib/taos/mnode_bak/wal/"
# new taos client
conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn1)
cur1 = conn1.cursor()
tdSql.init(cur1, True)
# new db ,new super tables , child tables, and insert data
tdSql.execute("drop database if exists db2")
os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath)
tdSql.execute("drop database if exists db1")
os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath)
tdSql.execute("drop table if exists db2.stb0")
os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath)
# new general tables and modify general tables;
tdSql.execute("use db2")
tdSql.execute("drop table if exists stb1_0")
tdSql.execute("drop table if exists stb1_1")
tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
tdSql.execute("alter table db2.stb0 add column c4 int")
tdSql.execute("alter table db2.stb0 drop column c2")
tdSql.execute("alter table db2.stb0 add tag t3 int")
tdSql.execute("alter table db2.stb0 drop tag t1")
tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ")
tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
tdSql.execute("alter table stb2_0 add column c2 binary(4)")
tdSql.execute("alter table stb2_0 drop column c1")
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
# create db utest and modify super tables;
dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"]
tdSql.execute("drop database if exists utest")
tdSql.execute("create database utest keep 3650")
tdSql.execute("use utest")
tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
rowNum2= 988
for i in range(rowNum2):
tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) )
rowNum3= 988
for i in range(rowNum3):
tdSql.execute("alter table test drop column col%d ;" %( i+14) )
self.rowNum = 1
self.rowNum2 = 100
self.ts = 1537146000000
for j in range(self.rowNum2):
tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
for i in range(self.rowNum):
tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
% (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
# delete child tables;
for j in range(self.rowNum2):
tdSql.execute("drop table if exists test%d" % (j+1))
#restart taosd
os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
sleep(20)
print("123")
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
sleep(4)
tdSql.execute("reset query cache")
query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
print(query_pid2)
# new taos connecting to server
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn2)
cur2 = conn2.cursor()
tdSql.init(cur2, True)
# check data correct
tdSql.query("show databases")
tdSql.execute("use db2")
tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 1)
tdSql.query("select count (tbname) from stb1")
tdSql.checkRows(0)
tdSql.query("select count(*) from stb0_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select * from stb2_0")
tdSql.checkData(1, 2, 'R')
tdSql.execute("use utest")
tdSql.query("select count (tbname) from test")
tdSql.checkData(0, 0, 1)
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf tsdb/%s.sql" % testcaseFilename )
clients = TwoClients()
clients.initConnection()
# clients.getBuildPath()
clients.run()
\ No newline at end of file
tests/pytest/tsdb/tsdbCompClusterReplica2.py 0 → 100644
浏览文件 @ 9f96559a
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import sys
sys.path.insert(0, os.getcwd())
from util.log import *
from util.sql import *
from util.dnodes import *
import taos
import threading
import subprocess
from random import choice
class TwoClients:
def initConnection(self):
self.host = "chenhaoran02"
self.user = "root"
self.password = "taosdata"
self.config = "/etc/taos/"
self.port =6030
self.rowNum = 10
self.ts = 1537146000000
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def run(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath+ "/build/bin/"
walFilePath = "/var/lib/taos/mnode_bak/wal/"
# new taos client
conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn1)
cur1 = conn1.cursor()
tdSql.init(cur1, True)
# new db ,new super tables , child tables, and insert data
tdSql.execute("drop database if exists db2")
os.system("%staosdemo -f tsdb/insertDataDb1Replica2.json -y " % binPath)
tdSql.execute("drop database if exists db1")
os.system("%staosdemo -f tsdb/insertDataDb2Replica2.json -y " % binPath)
tdSql.execute("drop table if exists db2.stb0")
os.system("%staosdemo -f tsdb/insertDataDb2NewstabReplica2.json -y " % binPath)
# new general tables and modify general tables;
tdSql.execute("use db2")
tdSql.execute("drop table if exists stb1_0")
tdSql.execute("drop table if exists stb1_1")
tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
tdSql.execute("alter table db2.stb0 add column c4 int")
tdSql.execute("alter table db2.stb0 drop column c2")
tdSql.execute("alter table db2.stb0 add tag t3 int")
tdSql.execute("alter table db2.stb0 drop tag t1")
tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ")
tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
tdSql.execute("alter table stb2_0 add column c2 binary(4)")
tdSql.execute("alter table stb2_0 drop column c1")
tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
# create db utest replica 2 and modify super tables;
dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"]
tdSql.execute("drop database if exists utest")
tdSql.execute("create database utest keep 3650 replica 2 ")
tdSql.execute("use utest")
tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
rowNum2= 988
for i in range(rowNum2):
tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) )
rowNum3= 988
for i in range(rowNum3):
tdSql.execute("alter table test drop column col%d ;" %( i+14) )
self.rowNum = 1
self.rowNum2 = 100
self.ts = 1537146000000
for j in range(self.rowNum2):
tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
for i in range(self.rowNum):
tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
% (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
# delete child tables;
for j in range(self.rowNum2):
tdSql.execute("drop table if exists test%d" % (j+1))
# drop dnodes and restart taosd;
sleep(3)
tdSql.execute(" drop dnode 'chenhaoran02:6030'; ")
sleep(20)
os.system("rm -rf /var/lib/taos/*")
print("clear dnode chenhaoran02'data files")
os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
print("start taosd")
sleep(10)
tdSql.execute("reset query cache ;")
tdSql.execute("create dnode chenhaoran02 ;")
# #
# os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2")
# sleep(20)
# os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &")
# sleep(4)
# tdSql.execute("reset query cache")
# query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
# print(query_pid2)
# new taos connecting to server
conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
print(conn2)
cur2 = conn2.cursor()
tdSql.init(cur2, True)
# check data correct
tdSql.query("show databases")
tdSql.execute("use db2")
tdSql.query("select count (tbname) from stb0")
tdSql.checkData(0, 0, 1)
tdSql.query("select count (tbname) from stb1")
tdSql.checkRows(0)
tdSql.query("select count(*) from stb0_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb0")
tdSql.checkData(0, 0, 2)
tdSql.query("select count(*) from stb2_0")
tdSql.checkData(0, 0, 2)
tdSql.query("select * from stb2_0")
tdSql.checkData(1, 2, 'R')
tdSql.execute("use utest")
tdSql.query("select count (tbname) from test")
tdSql.checkData(0, 0, 1)
# delete useless file
testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf ./insert_res.txt")
os.system("rm -rf tsdb/%s.sql" % testcaseFilename )
clients = TwoClients()
clients.initConnection()
# clients.getBuildPath()
clients.run()
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或