diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b954db0734be2ffa14a01955fc50e1573bc33ee5..3dbf74eaa7fbf37a8c94faf4787a8082cc4fc0ed 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -46,7 +46,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8d6d9baca30b767ed70c49a41d478556627732cf..b1cfbcedcf1d729ab2192f0b5c57c00140887ec9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4800,16 +4800,15 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea } static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { - pMsg->blocksPerTable = htons(pCreateDb->numOfBlocksPerTable); - pMsg->compression = pCreateDb->compressionLevel; - - pMsg->commitLog = (char)pCreateDb->commitLog; - pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); - pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock); + pMsg->cacheBlockSize = htonl(-1); + pMsg->totalBlocks = htonl(-1); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl(pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(-1); + pMsg->maxRowsPerFileBlock = htonl(-1); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->replications = pCreateDb->replica; pMsg->ignoreExist = pCreateDb->ignoreExists; } @@ -5342,29 +5341,22 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { char msg[512] = {0}; - if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 2)) { + if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->replications != -1 && - (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { + (pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->rowsInFileBlock); - if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { - snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5376,9 +5368,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { } val = htonl(pCreate->maxSessions); - if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { + if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + TSDB_MIN_TABLES, TSDB_MAX_TABLES); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5388,24 +5380,17 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", - pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4320e8f81b39d521bba29750d709e60709d33a4b..0eff4878bf5811cd5903a210b2036945fa051b37 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -131,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; tsem_init(&pSql->rspSem, 0, 0); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index f770c8d38ff34252f4e5d70c5d9b11301e2fd03a..69829f2ba4c4d5d76ce56bb4af8fc6b98511a110 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -74,11 +74,13 @@ extern int16_t tsNumOfVnodesPerCore; extern int16_t tsNumOfTotalVnodes; extern uint32_t tsPublicIpInt; -extern int32_t tsMaxCacheSize; -extern int32_t tsSessionsPerVnode; +extern int32_t tsCacheBlockSize; +extern int32_t tsTotalBlocks; +extern int32_t tsTablesPerVnode; extern int16_t tsDaysPerFile; extern int32_t tsDaysToKeep; -extern int32_t tsRowsInFileBlock; +extern int32_t tsMinRowsInFileBlock; +extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; extern int16_t tsCompression; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 5aa50677c9a5e6f03028c2d17f764ef2a4f85989..9124295fa065c11f336ecd2826837565989d9915 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -85,20 +85,22 @@ int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; #ifdef _TD_ARM_32_ -int32_t tsSessionsPerVnode = 100; +int32_t tsTablesPerVnode = 100; #else -int32_t tsSessionsPerVnode = 1000; +int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES; #endif -int32_t tsMaxCacheSize = 64; //64M -int16_t tsDaysPerFile = 10; -int32_t tsDaysToKeep = 3650; -int32_t tsRowsInFileBlock = 4096; -int16_t tsCommitTime = 3600; // seconds -int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; -int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; -int16_t tsCommitLog = 1; -int32_t tsReplications = TSDB_REPLICA_MIN_NUM; +int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; +int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; +int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; +int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; +int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; +int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; +int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds +int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; +int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL; +int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; /** * Change the meaning of affected rows: @@ -567,16 +569,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "ctime"; - cfg.ptr = &tsCommitTime; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 30; - cfg.maxValue = 40960; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); - cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -678,32 +670,42 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); // database configs - cfg.option = "clog"; - cfg.ptr = &tsCommitLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "tables"; + cfg.ptr = &tsTablesPerVnode; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_TABLES; + cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "cache"; + cfg.ptr = &tsCacheBlockSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; + cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + + cfg.option = "blocks"; + cfg.ptr = &tsTotalBlocks; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; + cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365; + cfg.minValue = TSDB_MIN_DAYS_PER_FILE; + cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -712,48 +714,68 @@ static void doInitGlobalConfig() { cfg.ptr = &tsDaysToKeep; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365000; + cfg.minValue = TSDB_MIN_KEEP; + cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "replica"; - cfg.ptr = &tsReplications; + cfg.option = "minRows"; + cfg.ptr = &tsMinRowsInFileBlock; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; + cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "tables"; - cfg.ptr = &tsSessionsPerVnode; + cfg.option = "maxRows"; + cfg.ptr = &tsMaxRowsInFileBlock; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES_PER_VNODE; - cfg.maxValue = TSDB_MAX_TABLES_PER_VNODE; + cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "cache"; - cfg.ptr = &tsMaxCacheSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.option = "ctime"; + cfg.ptr = &tsCommitTime; + cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; + cfg.minValue = TSDB_MIN_COMMIT_TIME; + cfg.maxValue = TSDB_MAX_COMMIT_TIME; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; + cfg.unitType = TAOS_CFG_UTYPE_SECOND; + taosInitConfigOption(cfg); + + cfg.option = "comp"; + cfg.ptr = &tsCompression; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_COMP_LEVEL; + cfg.maxValue = TSDB_MAX_COMP_LEVEL; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "clog"; + cfg.ptr = &tsCommitLog; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_CLOG_LEVEL; + cfg.maxValue = TSDB_MAX_CLOG_LEVEL; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "rows"; - cfg.ptr = &tsRowsInFileBlock; + cfg.option = "replica"; + cfg.ptr = &tsReplications; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 200; - cfg.maxValue = 1048576; + cfg.minValue = TSDB_MIN_REPLICA_NUM; + cfg.maxValue = TSDB_MAX_REPLICA_NUM; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 97768ca7439906dd4dc7c62f3c9d212b237a9af0..a319c0bb9a733febe4c093159e6c3749f7878b4f 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -129,13 +129,14 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.maxCacheSize = htobe64(pCreate->cfg.maxCacheSize); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 043e58cb35e34d4b6d5791322dcb9736f6250bf4..1a3316cdcf8ad3aee106a7efd641f55f6ef37795 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -221,44 +221,56 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_MPEERS 5 #define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) -#define TSDB_REPLICA_MIN_NUM 1 -#define TSDB_REPLICA_MAX_NUM 3 - #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta -//default value == 10 -#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days -#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. -#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years +#define TSDB_MIN_TOTAL_BLOCKS 2 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 2 -#define TSDB_MIN_COMPRESSION_LEVEL 0 -#define TSDB_MAX_COMPRESSION_LEVEL 2 +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 200000 +#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 -#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 -#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_MIN_CACHE_BLOCKS 100 -#define TSDB_MAX_CACHE_BLOCKS 409600 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_MIN_AVG_BLOCKS 2 -#define TSDB_MAX_AVG_BLOCKS 2048 -#define TSDB_DEFAULT_AVG_BLOCKS 4 +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -/* - * There is a bug in function taosAllocateId. - * When "create database tables 1" is executed, the wrong sid is assigned, so the minimum value is set to 2. - */ -#define TSDB_MIN_TABLES_PER_VNODE 2 -#define TSDB_MAX_TABLES_PER_VNODE 220000 +#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_CLOG_LEVEL 0 +#define TSDB_MAX_CLOG_LEVEL 2 +#define TSDB_DEFAULT_CLOG_LEVEL 2 + +#define TSDB_MIN_REPLICA_NUM 1 +#define TSDB_MAX_REPLICA_NUM 3 +#define TSDB_DEFAULT_REPLICA_NUM 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 8e6e762017ec972c127a3fa399fe668bf128e0fc..54523196dd54c9b3d3ff104e5cfe1a293455bd4d 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -517,27 +517,21 @@ typedef struct { typedef struct { char acct[TSDB_USER_LEN + 1]; char db[TSDB_DB_NAME_LEN + 1]; - uint32_t vgId; int32_t maxSessions; - int32_t cacheBlockSize; - union { - int32_t totalBlocks; - float fraction; - } cacheNumOfBlocks; - int32_t daysPerFile; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t daysToKeep; - int32_t commitTime; - int32_t rowsInFileBlock; - int16_t blocksPerTable; - int8_t compression; - int8_t commitLog; - int8_t replications; - int8_t repStrategy; - int8_t loadLatest; // load into mem or not - uint8_t precision; // time resolution - int8_t ignoreExist; + int32_t cacheBlockSize; //MB + int32_t totalBlocks; + int32_t daysPerFile; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t daysToKeep; + int32_t commitTime; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int8_t compression; + int8_t commitLog; + int8_t replications; + uint8_t precision; // time resolution + int8_t ignoreExist; } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { @@ -605,20 +599,21 @@ typedef struct { typedef struct { uint32_t vgId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; - int64_t maxCacheSize; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; - uint8_t precision; // time resolution + int8_t precision; int8_t compression; - int8_t wals; int8_t commitLog; int8_t replications; + int8_t wals; int8_t quorum; uint32_t arbitratorIp; int8_t reserved[16]; @@ -653,7 +648,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; int8_t numOfIps; - SIpAddr ipAddr[TSDB_REPLICA_MAX_NUM]; + SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; } SCMVgroupInfo; typedef struct { @@ -697,7 +692,7 @@ typedef struct { } SVnodeDesc; typedef struct { - SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM]; + SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM]; int16_t index; // used locally int32_t vgId; int32_t numOfSids; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index e5c884c81931f4b1977cc9413f1186dbaf25cdd3..38ded5cf10720b7b0f4443e22cd4146da0ab3f7e 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -45,15 +45,19 @@ typedef struct { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { - int8_t precision; - int8_t compression; int32_t tsdbId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy + int32_t keep; // day of data to keep + int32_t keep1; + int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t keep; // day of data to keep - int64_t maxCacheSize; // maximum cache size this TSDB can use + int32_t commitTime; + int8_t precision; + int8_t compression; } STsdbCfg; void tsdbSetDefaultCfg(STsdbCfg *pCfg); diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 6cbfc72e625ba5cd316d0c5ab14fae681ef141ee..f91a8d6bb4c98c1abb88ce5a3367af22445ccb89 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -143,14 +143,15 @@ typedef struct SVgObj { } SVgObj; typedef struct { - int64_t maxCacheSize; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; // minimum rows per file block - int32_t maxRowsPerFileBlock; // maximum rows per file block + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; int8_t precision; int8_t compression; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index b3424ffd63fccaff77f24cb7c33b2acd93caf32e..844d40b6056c23e9d59e03e010c9b5279026c3ee 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -177,78 +177,81 @@ SDbObj *mgmtGetDbByTableId(char *tableId) { } static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - mError("invalid db option maxCacheSize:%d valid range: [%d, %d]", pCfg->maxCacheSize, TSDB_MIN_CACHE_BLOCK_SIZE, + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); } - if (pCfg->maxTables < TSDB_MIN_TABLES_PER_VNODE || pCfg->maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS, + TSDB_MAX_TOTAL_BLOCKS); + } + + if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { - mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, - TSDB_FILE_MAX_PARTITION_RANGE); + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysToKeep1 < pCfg->daysPerFile) { - mError("invalid db option daystokeep:%d", pCfg->daysToKeep); + if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) { + mError("invalid db option daysToKeep:%d", pCfg->daysToKeep); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep2 > pCfg->daysToKeep || pCfg->daysToKeep2 < pCfg->daysToKeep1) { - mError("invalid db option daystokeep1:%d, daystokeep2:%d, daystokeep:%d", pCfg->daysToKeep1, - pCfg->daysToKeep2, pCfg->daysToKeep); + if (pCfg->daysToKeep < pCfg->daysPerFile) { + mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile); return TSDB_CODE_INVALID_OPTION; } - - if (pCfg->minRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < pCfg->minRowsPerFileBlock) { + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCfg->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { - mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, - TSDB_MAX_COMMIT_TIME_INTERVAL); + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->precision != TSDB_TIME_PRECISION_MILLI && pCfg->precision != TSDB_TIME_PRECISION_MICRO) { - mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + if (pCfg->precision != TSDB_MIN_PRECISION && pCfg->precision != TSDB_MAX_PRECISION) { + mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION, + TSDB_MAX_PRECISION); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->compression < TSDB_MIN_COMPRESSION_LEVEL || pCfg->compression > TSDB_MAX_COMPRESSION_LEVEL) { - mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMP_LEVEL, + TSDB_MAX_COMP_LEVEL); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitLog < 0 || pCfg->commitLog > 2) { + if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) { mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->replications < TSDB_REPLICA_MIN_NUM || pCfg->replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); + if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, + TSDB_MAX_REPLICA_NUM); return TSDB_CODE_INVALID_OPTION; } @@ -256,14 +259,15 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { } static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < 0) pCfg->maxCacheSize = tsMaxCacheSize; - if (pCfg->maxTables < 0) pCfg->maxTables = tsSessionsPerVnode; + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks; + if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile; if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; - if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsRowsInFileBlock; - if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = pCfg->minRowsPerFileBlock * 2; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock; if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; if (pCfg->compression < 0) pCfg->compression = tsCompression; @@ -293,14 +297,15 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); pDb->createdTime = taosGetTimestampMs(); pDb->cfg = (SDbCfg) { - .maxCacheSize = 64,//(int64_t)pCreate->cacheBlockSize * pCreate->cacheNumOfBlocks.totalBlocks, + .cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, .maxTables = pCreate->maxSessions, .daysPerFile = pCreate->daysPerFile, .daysToKeep = pCreate->daysToKeep, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRowsPerFileBlock = pCreate->rowsInFileBlock * 1, - .maxRowsPerFileBlock = pCreate->rowsInFileBlock * 2, + .minRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, .commitTime = pCreate->commitTime, .precision = pCreate->precision, .compression = pCreate->compression, @@ -459,13 +464,25 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "rows"); + strcpy(pSchema[cols].name, "cache(MB)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "blocks"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(Mb)"); + strcpy(pSchema[cols].name, "maxrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -492,7 +509,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 3; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "time precision"); + strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -584,13 +601,21 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxCacheSize; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.commitTime; cols++; @@ -664,14 +689,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); pCreate->daysToKeep = htonl(pCreate->daysToKeep); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->blocksPerTable = htons(pCreate->blocksPerTable); - pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); - + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + int32_t code; if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_GRANT_EXPIRED; @@ -688,9 +714,9 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { - SDbCfg newCfg = pDb->cfg; - int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxTables = htonl(pAlter->maxSessions); + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxTables = htonl(pAlter->maxSessions); int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; @@ -702,8 +728,8 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (replications > 0 && replications != pDb->cfg.replications) { mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); - if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + if (replications < TSDB_MIN_REPLICA_NUM || replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); terrno = TSDB_CODE_INVALID_OPTION; } newCfg.replications = replications; @@ -711,8 +737,8 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); - if (maxTables < TSDB_MIN_TABLES_PER_VNODE || maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + if (maxTables < TSDB_MIN_TABLES || maxTables > TSDB_MAX_TABLES) { + mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); terrno = TSDB_CODE_INVALID_OPTION; } if (maxTables < pDb->cfg.maxTables) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index e053ea66f5c0ca9073231e8484cce7107682def7..feedf5fdc3b9148e3a31d0e2c8accc27a5cf53d9 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -535,23 +535,21 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); + pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); + pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(pDb->cfg.maxTables); - pCfg->maxCacheSize = htobe64(pDb->cfg.maxCacheSize); - pCfg->maxCacheSize = htobe64(-1); //TODO - pCfg->minRowsPerFileBlock = htonl(-1); //TODO - pCfg->maxRowsPerFileBlock = htonl(-1); //TODO pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); - pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); - pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); - pCfg->daysToKeep = htonl(-1); //TODO + pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); + pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); + pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock); + pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock); pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; - pCfg->compression = -1; - pCfg->wals = 3; pCfg->commitLog = pDb->cfg.commitLog; pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->wals = 3; pCfg->quorum = 1; SMDVnodeDesc *pNodes = pVnode->nodes; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5fd82956f79a7836864292c2c1058a230470ae4c..d4ed7a6384c694ec63b4b81f2b15318e7e89368a 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -129,9 +129,6 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); char * getTupleKey(const void *data); -// ------------------------------ TSDB CACHE INTERFACES ------------------------------ -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ - typedef struct { int blockId; int offset; diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 942afd70a6875215686dcbe67d3cc551fc9b60ba..3e241773ed27204a4525254b61e07796b395ce02 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -26,6 +26,9 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) { if (pCache == NULL) return NULL; if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + cacheBlockSize *= (1024 * 1024); + + if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS; pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 289ad0f0d0cecf20fee9f605ab7af6362994eae6..ea1bc84d65a17355ac5878f30f9e2f96eec5b9b9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1,4 +1,5 @@ #include "os.h" +#include "taosdef.h" #include "tulog.h" #include "talgo.h" #include "tsdb.h" @@ -11,24 +12,6 @@ #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 100000 -#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 60 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_KEEP 3650 -#define TSDB_MIN_KEEP 1 -#define TSDB_MAX_KEEP INT_MAX -#define TSDB_DEFAULT_CACHE_SIZE (16 * 1024 * 1024) // 16M -#define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024) // 4M -#define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024) // 1G #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" @@ -70,7 +53,6 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) { pCfg->minRowsPerFileBlock = -1; pCfg->maxRowsPerFileBlock = -1; pCfg->keep = -1; - pCfg->maxCacheSize = -1; pCfg->compression = TWO_STAGE_COMP; } @@ -192,7 +174,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo); + pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -612,13 +594,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1; } - // Check maxCacheSize - if (pCfg->maxCacheSize == -1) { - pCfg->maxCacheSize = TSDB_DEFAULT_CACHE_SIZE; - } else { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_SIZE) return -1; - } - return 0; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 26791ae77c34b41e56d9b695885bc2df61a644b1..153bdbffd806f4addc6aa68b9f445f5a9aab40c6 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.maxCacheSize = pVnodeCfg->cfg.maxCacheSize; - + char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); @@ -390,26 +389,27 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - - len += snprintf(content + len, maxLen - len, " \"maxCacheSize\": %" PRId64 ",\n", pVnodeCfg->cfg.maxCacheSize); - + len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"commitLog\": %d,\n", pVnodeCfg->cfg.commitLog); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); + uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp; sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr); - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); @@ -463,19 +463,19 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; + pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); if (!maxTables || maxTables->type != cJSON_Number) { @@ -484,13 +484,34 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxTables = maxTables->valueint; - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode, pVnode->vgId); goto PARSE_OVER; } pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode, pVnode->vgId); @@ -505,19 +526,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + if (!commitTime || commitTime->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, commitTime not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.keep = daysToKeep->valueint; + pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - cJSON *maxCacheSize = cJSON_GetObjectItem(root, "maxCacheSize"); - if (!maxCacheSize || maxCacheSize->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, maxCacheSize not found", pVnode, pVnode->vgId); + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.maxCacheSize = maxCacheSize->valueint; + pVnode->tsdbCfg.compression = (int8_t)compression->valueint; cJSON *commitLog = cJSON_GetObjectItem(root, "commitLog"); if (!commitLog || commitLog->type != cJSON_Number) { @@ -534,12 +562,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { pVnode->walCfg.wals = (int8_t)wals->valueint; pVnode->walCfg.keep = 0; - cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); - if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); + pVnode->syncCfg.replica = (int8_t)replica->valueint; cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -548,12 +576,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); + cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); + if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.replica = (int8_t)replica->valueint; + pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index 092eebdbdffca2974b053b9066d084c4995adb24..cec8da67bbe3211cb2d5988c8724baaf68e6791f 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,6 +1,8 @@ run general/http/restful.sim -run general/http/restful_full.sim +run general/http/restful_insert.sim +#run general/http/restful_limit.sim +#run general/http/restful_full.sim +#run general/http/prepare.sim run general/http/telegraf.sim -run general/http/prepare.sim run general/http/grafana_bug.sim run general/http/grafana.sim