diff --git a/.codecov.yml b/.codecov.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c3ba81f9e2e9635e06fddd967f8d5bf364b14f0a
--- /dev/null
+++ b/.codecov.yml
@@ -0,0 +1,12 @@
+coverage:
+ precision: 2
+ notify:
+ email:
+ default:
+ to:
+ - sdsang@taosdata.com
+ - &author
+ only_pulls: false
+ layout: reach, diff, flags, files
+ flags: null
+ paths: null
diff --git a/.travis.yml b/.travis.yml
index bda5c0d75896ffacae6a048ab97ee34229b5e58b..73d2af4d025790936025819a1b600452fc01158e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -146,8 +146,8 @@ matrix:
fi
cd ${TRAVIS_BUILD_DIR}
- lcov -d . --capture -o coverage.info
- lcov -l coverage.info || exit $?
+ lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
+ lcov -l --rc lcov_branch_coverage=1 coverage.info || exit $?
gem install coveralls-lcov
@@ -166,7 +166,7 @@ matrix:
exit $?
fi
- bash <(curl -s https://codecov.io/bash) -f coverage.info
+ bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
if [ "$?" -eq "0" ]; then
echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
else
diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h
index 04fa7dcc7dd5e1cb8498b4fa4eb033dfcc92e7c3..17aa19cce7be5d5a35707ef68cb297c7ba6889a0 100644
--- a/src/common/inc/dataformat.h
+++ b/src/common/inc/dataformat.h
@@ -111,7 +111,6 @@ typedef struct SDataCol {
int len;
int offset;
void * pData; // Original data
- void * pCData; // Compressed data
} SDataCol;
typedef struct {
@@ -133,9 +132,12 @@ typedef struct {
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
void tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
+SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);
+int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
+void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
#ifdef __cplusplus
}
diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c
index 45850d1788d55cf25796d0d51c9518db38bf126a..fb20892452641e932dbbcf1d6ea9902efa091b2a 100644
--- a/src/common/src/dataformat.c
+++ b/src/common/src/dataformat.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
#include "dataformat.h"
+#include "tutil.h"
static int tdFLenFromSchema(STSchema *pSchema);
@@ -338,6 +339,28 @@ void tdFreeDataCols(SDataCols *pCols) {
}
}
+SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
+ SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints);
+ if (pRet == NULL) return NULL;
+
+ pRet->numOfCols = pDataCols->numOfCols;
+ pRet->sversion = pDataCols->sversion;
+ if (keepData) pRet->numOfPoints = pDataCols->numOfPoints;
+
+ for (int i = 0; i < pDataCols->numOfCols; i++) {
+ pRet->cols[i].type = pDataCols->cols[i].type;
+ pRet->cols[i].colId = pDataCols->cols[i].colId;
+ pRet->cols[i].bytes = pDataCols->cols[i].bytes;
+ pRet->cols[i].len = pDataCols->cols[i].len;
+ pRet->cols[i].offset = pDataCols->cols[i].offset;
+ pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
+
+ if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].bytes * pDataCols->numOfPoints);
+ }
+
+ return pRet;
+}
+
void tdResetDataCols(SDataCols *pCols) {
pCols->numOfPoints = 0;
for (int i = 0; i < pCols->maxCols; i++) {
@@ -382,6 +405,58 @@ static int tdFLenFromSchema(STSchema *pSchema) {
return ret;
}
-int tdMergeDataCols(SDataCols *target, SDataCols *source) {
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
+ ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
+
+ SDataCols *pTarget = tdDupDataCols(target, true);
+ if (pTarget == NULL) goto _err;
+ // tdResetDataCols(target);
+
+ int iter1 = 0;
+ int iter2 = 0;
+ tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
+
+ tdFreeDataCols(pTarget);
return 0;
+
+_err:
+ tdFreeDataCols(pTarget);
+ return -1;
+}
+
+void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
+ tdResetDataCols(target);
+
+ while (target->numOfPoints < tRows) {
+ if (*iter1 >= src1->numOfPoints && *iter2 >= src2->numOfPoints) break;
+
+ TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
+ TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+
+ if (key1 < key2) {
+ for (int i = 0; i < src1->numOfCols; i++) {
+ ASSERT(target->cols[i].type == src1->cols[i].type);
+ memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
+ (void *)((char *)(src1->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * (*iter1)),
+ TYPE_BYTES[target->cols[i].type]);
+ target->cols[i].len += TYPE_BYTES[target->cols[i].type];
+ }
+
+ target->numOfPoints++;
+ (*iter1)++;
+ } else if (key1 > key2) {
+ for (int i = 0; i < src2->numOfCols; i++) {
+ ASSERT(target->cols[i].type == src2->cols[i].type);
+ memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
+ (void *)((char *)(src2->cols[i].pData) + TYPE_BYTES[src2->cols[i].type] * (*iter2)),
+ TYPE_BYTES[target->cols[i].type]);
+ target->cols[i].len += TYPE_BYTES[target->cols[i].type];
+ }
+
+ target->numOfPoints++;
+ (*iter2)++;
+ } else {
+ ASSERT(false);
+ }
+ }
}
\ No newline at end of file
diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt
index 5735e1a8c18677f7323db6942d115fca681ab105..af2dc2d777bb6d294beecf3b3fa2210ab7230571 100644
--- a/src/dnode/CMakeLists.txt
+++ b/src/dnode/CMakeLists.txt
@@ -27,7 +27,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ENDIF ()
IF (TD_SYNC)
- TARGET_LINK_LIBRARIES(taosd replica sync)
+ TARGET_LINK_LIBRARIES(taosd balance sync)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c
index 90a093560f7860a3414f803c7549f50f4388797f..38be318c2570a89db07ef86225ffbcec409f3cbc 100644
--- a/src/dnode/src/dnodeMClient.c
+++ b/src/dnode/src/dnodeMClient.c
@@ -23,12 +23,13 @@
#include "tsync.h"
#include "ttime.h"
#include "ttimer.h"
-#include "treplica.h"
+#include "tbalance.h"
+#include "vnode.h"
+#include "mnode.h"
#include "dnode.h"
#include "dnodeMClient.h"
#include "dnodeModule.h"
#include "dnodeMgmt.h"
-#include "vnode.h"
#define MPEER_CONTENT_LEN 2000
@@ -181,7 +182,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
tsMnodeInfos.nodeInfos[i].nodeName);
}
dnodeSaveMnodeIpList();
- replicaNotify();
+ sdbUpdateSync();
}
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
diff --git a/src/inc/mnode.h b/src/inc/mnode.h
index e7ad88d6b66398e14411bcbf9e74ffeedb18cbda..35f7650c20e344f42ba8821fc0abeba50d035fe1 100644
--- a/src/inc/mnode.h
+++ b/src/inc/mnode.h
@@ -24,6 +24,7 @@ int32_t mgmtInitSystem();
int32_t mgmtStartSystem();
void mgmtCleanUpSystem();
void mgmtStopSystem();
+void sdbUpdateSync();
#ifdef __cplusplus
}
diff --git a/src/inc/treplica.h b/src/inc/tbalance.h
similarity index 71%
rename from src/inc/treplica.h
rename to src/inc/tbalance.h
index 3abed1c4aa258bca9e538b561d3da5599ca3e48e..9ffa6332c60957b2e86f1ad6fec4c64c4a64f43f 100644
--- a/src/inc/treplica.h
+++ b/src/inc/tbalance.h
@@ -13,8 +13,8 @@
* along with this program. If not, see .
*/
-#ifndef TDENGINE_REPLICA_H
-#define TDENGINE_REPLICA_H
+#ifndef TDENGINE_BALANCE_H
+#define TDENGINE_BALANCE_H
#ifdef __cplusplus
extern "C" {
@@ -23,13 +23,12 @@ extern "C" {
struct SVgObj;
struct SDnodeObj;
-int32_t replicaInit();
-void replicaCleanUp();
-void replicaNotify();
-void replicaReset();
-int32_t replicaAllocVnodes(struct SVgObj *pVgroup);
-int32_t replicaForwardReqToPeer(void *pHead);
-int32_t replicaDropDnode(struct SDnodeObj *pDnode);
+int32_t balanceInit();
+void balanceCleanUp();
+void balanceNotify();
+void balanceReset();
+int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
+int32_t balanceDropDnode(struct SDnodeObj *pDnode);
#ifdef __cplusplus
}
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 8f4e1db590440c57025db858dcd39713b90a346b..059964eb0093e8357692bf9b77b905571340b4e3 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -46,6 +46,7 @@ typedef struct {
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
int8_t precision;
+ int8_t compression;
int32_t tsdbId;
int32_t maxTables; // maximum number of tables this repository can have
int32_t daysPerFile; // day per file sharding policy
@@ -60,13 +61,13 @@ STsdbCfg *tsdbCreateDefaultCfg();
void tsdbFreeCfg(STsdbCfg *pCfg);
// --------- TSDB REPOSITORY DEFINITION
-typedef void tsdb_repo_t; // use void to hide implementation details from outside
+typedef void TsdbRepoT; // use void to hide implementation details from outside
-int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
-int32_t tsdbDropRepo(tsdb_repo_t *repo);
-tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
-int32_t tsdbCloseRepo(tsdb_repo_t *repo);
-int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
+int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
+int32_t tsdbDropRepo(TsdbRepoT *repo);
+TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
+int32_t tsdbCloseRepo(TsdbRepoT *repo);
+int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg);
// --------- TSDB TABLE DEFINITION
typedef struct {
@@ -76,15 +77,15 @@ typedef struct {
// --------- TSDB TABLE configuration
typedef struct {
- ETableType type;
- char * name;
- STableId tableId;
- int32_t sversion;
- char * sname; // super table name
- int64_t superUid;
- STSchema * schema;
- STSchema * tagSchema;
- SDataRow tagValues;
+ ETableType type;
+ char * name;
+ STableId tableId;
+ int32_t sversion;
+ char * sname; // super table name
+ int64_t superUid;
+ STSchema * schema;
+ STSchema * tagSchema;
+ SDataRow tagValues;
} STableCfg;
int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid);
@@ -96,11 +97,11 @@ int tsdbTableSetName(STableCfg *config, char *name, bool dup);
int tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
void tsdbClearTableCfg(STableCfg *config);
-int32_t tsdbGetTableTagVal(tsdb_repo_t *repo, STableId id, int32_t col, int16_t* type, int16_t* bytes, char** val);
+int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val);
-int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
-int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
-int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
+int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
+int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
+int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
// the TSDB repository info
typedef struct STsdbRepoInfo {
@@ -110,7 +111,7 @@ typedef struct STsdbRepoInfo {
int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository
// TODO: Other informations to add
} STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
+STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo);
// the meter information report structure
typedef struct {
@@ -119,7 +120,7 @@ typedef struct {
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
+STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
// -- FOR INSERT DATA
/**
@@ -129,11 +130,11 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
*
* @return the number of points inserted, -1 for failure and the error number is set
*/
-int32_t tsdbInsertData(tsdb_repo_t *pRepo, SSubmitMsg *pMsg);
+int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg);
// -- FOR QUERY TIME SERIES DATA
-typedef void *tsdb_query_handle_t; // Use void to hide implementation details
+typedef void *TsdbQueryHandleT; // Use void to hide implementation details
typedef struct STableGroupList { // qualified table object list in group
SArray *pGroupList;
@@ -167,21 +168,21 @@ typedef struct SDataBlockInfo {
typedef struct {
size_t numOfTables;
- SArray* pGroupList;
+ SArray *pGroupList;
} STableGroupInfo;
typedef struct {
} SFields;
#define TSDB_TS_GREATER_EQUAL 1
-#define TSDB_TS_LESS_EQUAL 2
+#define TSDB_TS_LESS_EQUAL 2
typedef struct SQueryRowCond {
int32_t rel;
TSKEY ts;
} SQueryRowCond;
-typedef void *tsdbpos_t;
+typedef void *TsdbPosT;
/**
* Get the data block iterator, starting from position according to the query condition
@@ -189,14 +190,14 @@ typedef void *tsdbpos_t;
* @param pTableList table sid list
* @return
*/
-tsdb_query_handle_t *tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
+TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
/**
* move to next block
* @param pQueryHandle
* @return
*/
-bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle);
+bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
/**
* Get current data block information
@@ -204,7 +205,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle);
* @param pQueryHandle
* @return
*/
-SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle);
+SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle);
/**
*
@@ -216,7 +217,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle);
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
* @return
*/
-int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis);
+int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
/**
* The query condition with primary timestamp is passed to iterator during its constructor function,
@@ -226,7 +227,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SData
* @param pQueryHandle
* @return
*/
-SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList);
+SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pIdList);
/**
* todo remove the parameter of position, and order type
@@ -238,7 +239,7 @@ SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList
* @param order ascending order or descending order
* @return
*/
-int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order);
+int32_t tsdbResetQuery(TsdbQueryHandleT *pQueryHandle, STimeWindow *window, TsdbPosT position, int16_t order);
/**
* todo remove this function later
@@ -246,7 +247,7 @@ int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, t
* @param pIdList
* @return
*/
-SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
+SArray *tsdbRetrieveDataRow(TsdbQueryHandleT *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
/**
* Get iterator for super tables, of which tags values satisfy the tag filter info
@@ -259,7 +260,7 @@ SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList,
* @param pTagFilterStr tag filter info
* @return
*/
-tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
+TsdbQueryHandleT *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
/**
* Get the qualified tables for (super) table query.
@@ -269,7 +270,7 @@ tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stable
* @param pQueryHandle
* @return table sid list. the invoker is responsible for the release of this the sid list.
*/
-SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);
+SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
/**
* Get the qualified table id for a super table according to the tag query expression.
@@ -277,16 +278,16 @@ SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);
* @param pTagCond. tag query condition
*
*/
-int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupList,
- SColIndex* pColIndex, int32_t numOfCols);
+int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList,
+ SColIndex *pColIndex, int32_t numOfCols);
-int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pGroupInfo);
+int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo);
/**
* clean up the query handle
* @param queryHandle
*/
-void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle);
+void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
#ifdef __cplusplus
}
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index e54820dffdcbe80781aa3b8dd9301ec5f53723aa..e8a7a1458f63b586c91fb6c7999995ad6110d6b1 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -47,7 +47,6 @@ void* vnodeGetVnode(int32_t vgId); // keep refcount unchanged
void* vnodeGetRqueue(void *);
void* vnodeGetWqueue(int32_t vgId);
void* vnodeGetWal(void *pVnode);
-void* vnodeGetTsdb(void *pVnode);
int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
void vnodeBuildStatusMsg(void * param);
diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h
index 1faa616cebe12eddb000db0e97f731c33f7463d4..34b5d19cf21108aa7f9808bed4329fed02ebb9b5 100644
--- a/src/mnode/inc/mgmtMnode.h
+++ b/src/mnode/inc/mgmtMnode.h
@@ -39,10 +39,9 @@ int32_t mgmtGetMnodesNum();
void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
void mgmtReleaseMnode(struct SMnodeObj *pMnode);
-bool mgmtIsMaster();
-
+char * mgmtGetMnodeRoleStr();
void mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp);
-void mgmtGetMnodeList(void *mpeers);
+void mgmtGetMnodeList(void *mnodes);
#ifdef __cplusplus
}
diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h
index 8ecb5ef1524060f3081e667ea132f934e37ba250..c09d215adb8d29ef24ccd3046c8acd47e977721d 100644
--- a/src/mnode/inc/mgmtSdb.h
+++ b/src/mnode/inc/mgmtSdb.h
@@ -36,54 +36,47 @@ typedef enum {
SDB_KEY_STRING,
SDB_KEY_INT,
SDB_KEY_AUTO
-} ESdbKeyType;
+} ESdbKey;
typedef enum {
SDB_OPER_GLOBAL,
SDB_OPER_LOCAL
-} ESdbOperType;
+} ESdbOper;
typedef struct {
- ESdbOperType type;
- void * table;
- void * pObj;
- int32_t rowSize;
- void * rowData;
-} SSdbOperDesc;
+ ESdbOper type;
+ void * table;
+ void * pObj;
+ int32_t rowSize;
+ void * rowData;
+} SSdbOper;
typedef struct {
char *tableName;
int32_t hashSessions;
int32_t maxRowSize;
int32_t refCountPos;
- ESdbTable tableId;
- ESdbKeyType keyType;
- int32_t (*insertFp)(SSdbOperDesc *pOper);
- int32_t (*deleteFp)(SSdbOperDesc *pOper);
- int32_t (*updateFp)(SSdbOperDesc *pOper);
- int32_t (*encodeFp)(SSdbOperDesc *pOper);
- int32_t (*decodeFp)(SSdbOperDesc *pDesc);
- int32_t (*destroyFp)(SSdbOperDesc *pDesc);
+ ESdbTable tableId;
+ ESdbKey keyType;
+ int32_t (*insertFp)(SSdbOper *pOper);
+ int32_t (*deleteFp)(SSdbOper *pOper);
+ int32_t (*updateFp)(SSdbOper *pOper);
+ int32_t (*encodeFp)(SSdbOper *pOper);
+ int32_t (*decodeFp)(SSdbOper *pDesc);
+ int32_t (*destroyFp)(SSdbOper *pDesc);
int32_t (*restoredFp)();
} SSdbTableDesc;
-typedef struct {
- int64_t version;
- void * wal;
- pthread_mutex_t mutex;
-} SSdbObject;
-
int32_t sdbInit();
void sdbCleanUp();
-SSdbObject *sdbGetObj();
-
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
-int sdbProcessWrite(void *param, void *data, int type);
+bool sdbIsMaster();
+void sdbUpdateMnodeRoles();
-int32_t sdbInsertRow(SSdbOperDesc *pOper);
-int32_t sdbDeleteRow(SSdbOperDesc *pOper);
-int32_t sdbUpdateRow(SSdbOperDesc *pOper);
+int32_t sdbInsertRow(SSdbOper *pOper);
+int32_t sdbDeleteRow(SSdbOper *pOper);
+int32_t sdbUpdateRow(SSdbOper *pOper);
void *sdbGetRow(void *handle, void *key);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow);
diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c
index 3a52715274059f0b386810bcd8eb36a7f679b636..3e04399fe7f24384c9e459bd4adea27d7ee600e3 100644
--- a/src/mnode/src/mgmtAcct.c
+++ b/src/mnode/src/mgmtAcct.c
@@ -30,28 +30,28 @@ void * tsAcctSdb = NULL;
int32_t tsAcctUpdateSize;
static void mgmtCreateRootAcct();
-static int32_t mgmtActionAcctDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
pthread_mutex_destroy(&pAcct->mutex);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtAcctActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtAcctActionInsert(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
pthread_mutex_init(&pAcct->mutex, NULL);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtActionAcctDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtActionAcctDelete(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
mgmtDropAllUsers(pAcct);
mgmtDropAllDbs(pAcct);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtActionAcctUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtActionAcctUpdate(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
SAcctObj *pSaved = mgmtGetAcct(pAcct->user);
if (pAcct != pSaved) {
@@ -61,14 +61,14 @@ static int32_t mgmtActionAcctUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtActionActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtActionActionEncode(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
pOper->rowSize = tsAcctUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtActionAcctDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtActionAcctDecode(SSdbOper *pOper) {
SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj));
if (pAcct == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -110,7 +110,7 @@ int32_t mgmtInitAccts() {
return -1;
}
- mTrace("account table is created");
+ mTrace("table:accounts table is created");
return acctInit();
}
@@ -179,7 +179,7 @@ static void mgmtCreateRootAcct() {
pAcct->acctId = sdbGetId(tsAcctSdb);
pAcct->createdTime = taosGetTimestampMs();
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsAcctSdb,
.pObj = pAcct,
diff --git a/src/mnode/src/mgmtReplica.c b/src/mnode/src/mgmtBalance.c
similarity index 86%
rename from src/mnode/src/mgmtReplica.c
rename to src/mnode/src/mgmtBalance.c
index 05a303a69bb82497672176b62897cf50d013a4bc..8ca651be2c6bca764752fdae7f20c89037e5f292 100644
--- a/src/mnode/src/mgmtReplica.c
+++ b/src/mnode/src/mgmtBalance.c
@@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtMnode.h"
@@ -25,13 +25,12 @@
#ifndef _SYNC
-int32_t replicaInit() { return TSDB_CODE_SUCCESS; }
-void replicaCleanUp() {}
-void replicaNotify() {}
-void replicaReset() {}
-int32_t replicaForwardReqToPeer(void *pHead) { return TSDB_CODE_SUCCESS; }
+int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
+void balanceCleanUp() {}
+void balanceNotify() {}
+void balanceReset() {}
-int32_t replicaAllocVnodes(SVgObj *pVgroup) {
+int32_t balanceAllocVnodes(SVgObj *pVgroup) {
void * pNode = NULL;
SDnodeObj *pDnode = NULL;
SDnodeObj *pSelDnode = NULL;
diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c
index 4d8163deced31b5ab9260b43d3ffd07c025fe083..7d5b872f4a3949ed14197b217117829fdf7050a3 100644
--- a/src/mnode/src/mgmtDServer.c
+++ b/src/mnode/src/mgmtDServer.c
@@ -21,7 +21,7 @@
#include "tsystem.h"
#include "tutil.h"
#include "tgrant.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "tglobalcfg.h"
#include "dnode.h"
#include "mgmtDef.h"
diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c
index 9bf05ecfc4ac54e8e8ffaecd1fdfae77b1c3ba4c..90aaa03e9a9e495be597015a774fa06528733f10 100644
--- a/src/mnode/src/mgmtDb.c
+++ b/src/mnode/src/mgmtDb.c
@@ -46,12 +46,12 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg);
-static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionInsert(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
@@ -72,7 +72,7 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionDelete(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
@@ -84,7 +84,7 @@ static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SDbObj *pSaved = mgmtGetDb(pDb->name);
if (pDb != pSaved) {
@@ -94,14 +94,14 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionEncode(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtDbActionDecode(SSdbOper *pOper) {
SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj));
if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -146,7 +146,7 @@ int32_t mgmtInitDbs() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DB, mgmtGetDbMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mgmtRetrieveDbs);
- mTrace("db data is initialized");
+ mTrace("table:dbs table is created");
return 0;
}
@@ -318,7 +318,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = *pCreate;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@@ -671,7 +671,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
if (pDb->status) return TSDB_CODE_SUCCESS;
pDb->status = true;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@@ -756,7 +756,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@@ -814,7 +814,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) {
SDbObj *pDb = pMsg->pDb;
mPrint("db:%s, drop db from sdb", pDb->name);
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb
diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c
index 8b5969cfd019c18a7eda4d32d6da82e1eeb3c230..baec30942477f48fc6b35e6bf0655871d196a7a4 100644
--- a/src/mnode/src/mgmtDnode.c
+++ b/src/mnode/src/mgmtDnode.c
@@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tgrant.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "tglobalcfg.h"
#include "ttime.h"
#include "tutil.h"
@@ -52,12 +52,12 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mgmtDnodeActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDnodeActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
@@ -72,7 +72,7 @@ static int32_t mgmtDnodeActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
void * pNode = NULL;
void * pLastNode = NULL;
@@ -85,7 +85,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
if (pVgroup == NULL) break;
if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@@ -101,7 +101,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDnodeActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
SDnodeObj *pSaved = mgmtGetDnode(pDnode->dnodeId);
if (pDnode != pSaved) {
@@ -111,14 +111,14 @@ static int32_t mgmtDnodeActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDnodeActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionEncode(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize);
pOper->rowSize = tsDnodeUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtDnodeActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtDnodeActionDecode(SSdbOper *pOper) {
SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
if (pDnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -180,7 +180,7 @@ int32_t mgmtInitDnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes);
- mTrace("dnodes table is created");
+ mTrace("table:dnodes table is created");
return 0;
}
@@ -221,7 +221,7 @@ void mgmtReleaseDnode(SDnodeObj *pDnode) {
}
void mgmtUpdateDnode(SDnodeObj *pDnode) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
@@ -340,7 +340,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
- replicaNotify();
+ balanceNotify();
}
mgmtReleaseDnode(pDnode);
@@ -393,7 +393,7 @@ static int32_t mgmtCreateDnode(uint32_t ip) {
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1);
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
@@ -413,7 +413,7 @@ static int32_t mgmtCreateDnode(uint32_t ip) {
}
int32_t mgmtDropDnode(SDnodeObj *pDnode) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode
diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c
index 38f18b462ae69427088eea77bc7652ce9d144624..1d2f62a5935165ff9b6d4d0759a00882d36cbe5a 100644
--- a/src/mnode/src/mgmtMain.c
+++ b/src/mnode/src/mgmtMain.c
@@ -17,7 +17,7 @@
#include "os.h"
#include "taosdef.h"
#include "tsched.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "tgrant.h"
#include "ttimer.h"
#include "dnode.h"
@@ -62,7 +62,7 @@ int32_t mgmtStartSystem() {
}
if (grantInit() < 0) {
- mError("failed to init grants");
+ mError("failed to init grant");
return -1;
}
@@ -92,7 +92,7 @@ int32_t mgmtStartSystem() {
}
if (mgmtInitMnodes() < 0) {
- mError("failed to init mpeers");
+ mError("failed to init mnodes");
return -1;
}
@@ -101,8 +101,8 @@ int32_t mgmtStartSystem() {
return -1;
}
- if (replicaInit() < 0) {
- mError("failed to init replica")
+ if (balanceInit() < 0) {
+ mError("failed to init balance")
}
if (mgmtInitDClient() < 0) {
@@ -144,7 +144,7 @@ void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes();
- replicaCleanUp();
+ balanceCleanUp();
mgmtCleanUpShell();
mgmtCleanupDClient();
mgmtCleanupDServer();
@@ -161,7 +161,7 @@ void mgmtCleanUpSystem() {
}
void mgmtStopSystem() {
- if (mgmtIsMaster()) {
+ if (sdbIsMaster()) {
mTrace("it is a master mgmt node, it could not be stopped");
return;
}
diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c
index 8087ce5ad1513cdc89795fbe745bad28c96c96f6..bfb6480aec3eb8898b6bfd7b68255985999007b8 100644
--- a/src/mnode/src/mgmtMnode.c
+++ b/src/mnode/src/mgmtMnode.c
@@ -18,7 +18,7 @@
#include "taoserror.h"
#include "trpc.h"
#include "tsync.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "tutil.h"
#include "ttime.h"
#include "tsocket.h"
@@ -30,18 +30,17 @@
#include "mgmtShell.h"
#include "mgmtUser.h"
-int32_t tsMnodeIsMaster = true;
static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mgmtMnodeActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtMnodeActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
@@ -53,7 +52,7 @@ static int32_t mgmtMnodeActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtMnodeActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionDelete(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
@@ -65,7 +64,7 @@ static int32_t mgmtMnodeActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtMnodeActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SMnodeObj *pSaved = mgmtGetMnode(pMnode->mnodeId);
if (pMnode != pSaved) {
@@ -76,14 +75,14 @@ static int32_t mgmtMnodeActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtMnodeActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionEncode(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize);
pOper->rowSize = tsMnodeUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtMnodeActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
if (pMnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -133,7 +132,7 @@ int32_t mgmtInitMnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
- mTrace("mnodes table is created");
+ mTrace("table:mnodes table is created");
return TSDB_CODE_SUCCESS;
}
@@ -157,7 +156,7 @@ void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode);
}
-static char *mgmtGetMnodeRoleStr(int32_t role) {
+char *mgmtGetMnodeRoleStr(int32_t role) {
switch (role) {
case TAOS_SYNC_ROLE_OFFLINE:
return "offline";
@@ -172,8 +171,6 @@ static char *mgmtGetMnodeRoleStr(int32_t role) {
}
}
-bool mgmtIsMaster() { return tsMnodeIsMaster; }
-
void mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp) {
void *pNode = NULL;
while (1) {
@@ -213,10 +210,8 @@ void mgmtGetMnodeList(void *param) {
mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp);
mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort);
strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName);
- mPrint("node:%d role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
mnodes->inUse = index;
- mPrint("node:%d inUse:%d", pMnode->mnodeId, mnodes->inUse);
}
index++;
@@ -231,7 +226,7 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
pMnode->mnodeId = dnodeId;
pMnode->createdTime = taosGetTimestampMs();
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsMnodeSdb,
.pObj = pMnode,
@@ -252,7 +247,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
return TSDB_CODE_DNODE_NOT_EXIST;
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsMnodeSdb,
.pObj = pMnode
@@ -268,6 +263,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
}
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+ sdbUpdateMnodeRoles();
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;
diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c
index 4bc18d6a0de369bd417f4e8f9e4abdf7cc439e17..3344b2b759e5e74d8eb099434aabb8639bf2ee6f 100644
--- a/src/mnode/src/mgmtSdb.c
+++ b/src/mnode/src/mgmtSdb.c
@@ -18,65 +18,93 @@
#include "taoserror.h"
#include "tlog.h"
#include "trpc.h"
-#include "treplica.h"
+#include "tutil.h"
+#include "tbalance.h"
#include "tqueue.h"
#include "twal.h"
+#include "tsync.h"
#include "hashint.h"
#include "hashstr.h"
+#include "dnode.h"
+#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
+typedef enum {
+ SDB_ACTION_INSERT,
+ SDB_ACTION_DELETE,
+ SDB_ACTION_UPDATE
+} ESdbAction;
+
+typedef enum {
+ SDB_STATUS_OFFLINE,
+ SDB_STATUS_SERVING,
+ SDB_ACTION_CLOSING
+} ESdbStatus;
+
typedef struct _SSdbTable {
- char tableName[TSDB_DB_NAME_LEN + 1];
- ESdbTable tableId;
- ESdbKeyType keyType;
- int32_t hashSessions;
- int32_t maxRowSize;
- int32_t refCountPos;
- int32_t autoIndex;
- int64_t numOfRows;
- void * iHandle;
- int32_t (*insertFp)(SSdbOperDesc *pDesc);
- int32_t (*deleteFp)(SSdbOperDesc *pOper);
- int32_t (*updateFp)(SSdbOperDesc *pOper);
- int32_t (*decodeFp)(SSdbOperDesc *pOper);
- int32_t (*encodeFp)(SSdbOperDesc *pOper);
- int32_t (*destroyFp)(SSdbOperDesc *pOper);
- int32_t (*restoredFp)();
+ char tableName[TSDB_DB_NAME_LEN + 1];
+ ESdbTable tableId;
+ ESdbKey keyType;
+ int32_t hashSessions;
+ int32_t maxRowSize;
+ int32_t refCountPos;
+ int32_t autoIndex;
+ int64_t numOfRows;
+ void * iHandle;
+ int32_t (*insertFp)(SSdbOper *pDesc);
+ int32_t (*deleteFp)(SSdbOper *pOper);
+ int32_t (*updateFp)(SSdbOper *pOper);
+ int32_t (*decodeFp)(SSdbOper *pOper);
+ int32_t (*encodeFp)(SSdbOper *pOper);
+ int32_t (*destroyFp)(SSdbOper *pOper);
+ int32_t (*restoredFp)();
pthread_mutex_t mutex;
} SSdbTable;
+typedef struct {
+ ESyncRole role;
+ ESdbStatus status;
+ int64_t version;
+ void * sync;
+ void * wal;
+ SSyncCfg cfg;
+ sem_t sem;
+ int32_t code;
+ int32_t numOfTables;
+ SSdbTable *tableList[SDB_TABLE_MAX];
+ pthread_mutex_t mutex;
+} SSdbObject;
+
typedef struct {
int32_t rowSize;
void * row;
-} SRowMeta;
-
-typedef enum {
- SDB_ACTION_INSERT,
- SDB_ACTION_DELETE,
- SDB_ACTION_UPDATE
-} ESdbActionType;
-
-static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0};
-static int32_t tsSdbNumOfTables = 0;
-static SSdbObject * tsSdbObj;
+} SSdbRow;
+static SSdbObject tsSdbObj = {0};
static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash};
static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash};
static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash};
static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData};
static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash};
static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData};
+static int sdbWrite(void *param, void *data, int type);
-int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; }
-int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; }
+int32_t sdbGetId(void *handle) {
+ return ((SSdbTable *)handle)->autoIndex;
+}
+
+int64_t sdbGetNumOfRows(void *handle) {
+ return ((SSdbTable *)handle)->numOfRows;
+}
uint64_t sdbGetVersion() {
- if (tsSdbObj)
- return tsSdbObj->version;
- else
- return 0;
+ return tsSdbObj.version;
+}
+
+bool sdbIsMaster() {
+ return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER;
}
static char *sdbGetActionStr(int32_t action) {
@@ -106,26 +134,26 @@ static char *sdbGetkeyStr(SSdbTable *pTable, void *row) {
}
static void *sdbGetTableFromId(int32_t tableId) {
- return tsSdbTableList[tableId];
+ return tsSdbObj.tableList[tableId];
}
-int32_t sdbInit() {
- tsSdbObj = calloc(1, sizeof(SSdbObject));
- pthread_mutex_init(&tsSdbObj->mutex, NULL);
-
+static int32_t sdbInitWal() {
SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1};
- tsSdbObj->wal = walOpen(tsMnodeDir, &walCfg);
- if (tsSdbObj->wal == NULL) {
- sdbError("failed to open sdb in %s", tsMnodeDir);
+ tsSdbObj.wal = walOpen(tsMnodeDir, &walCfg);
+ if (tsSdbObj.wal == NULL) {
+ sdbError("failed to open sdb wal in %s", tsMnodeDir);
return -1;
}
- sdbTrace("open sdb file for read");
- walRestore(tsSdbObj->wal, tsSdbObj, sdbProcessWrite);
+ sdbTrace("open sdb wal for restore");
+ walRestore(tsSdbObj.wal, &tsSdbObj, sdbWrite);
+ return 0;
+}
+static void sdbRestoreTables() {
int32_t totalRows = 0;
int32_t numOfTables = 0;
- for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) {
+ for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) {
SSdbTable *pTable = sdbGetTableFromId(tableId);
if (pTable == NULL) continue;
if (pTable->restoredFp) {
@@ -134,23 +162,170 @@ int32_t sdbInit() {
totalRows += pTable->numOfRows;
numOfTables++;
- sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows);
+ sdbTrace("table:%s, is restored, numOfRows:%d", pTable->tableName, pTable->numOfRows);
+ }
+
+ sdbTrace("sdb is restored, version:%d totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
+}
+
+void sdbUpdateMnodeRoles() {
+ if (tsSdbObj.sync == NULL) return;
+
+ SNodesRole roles = {0};
+ syncGetNodesRole(tsSdbObj.sync, &roles);
+
+ mPrint("update mnodes:%d sync roles", tsSdbObj.cfg.replica);
+ for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) {
+ SMnodeObj *pMnode = mgmtGetMnode(roles.nodeId[i]);
+ if (pMnode != NULL) {
+ pMnode->role = roles.role[i];
+ mPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
+ mgmtReleaseMnode(pMnode);
+ }
+ }
+}
+
+static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
+ sdbUpdateMnodeRoles();
+ return 0;
+}
+
+static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
+ strcpy(name, "wal0");
+ return 0;
+}
+
+static void sdbNotifyRole(void *ahandle, int8_t role) {
+ mPrint("mnode role changed from %s to %s", mgmtGetMnodeRoleStr(tsSdbObj.role), mgmtGetMnodeRoleStr(role));
+
+ if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) {
+ balanceReset();
+ }
+ tsSdbObj.role = role;
+
+ sdbUpdateMnodeRoles();
+}
+
+static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
+ tsSdbObj.code = code;
+ sem_post(&tsSdbObj.sem);
+ mPrint("sdb forward request confirmed, result:%s", tstrerror(code));
+}
+
+static int32_t sdbForwardToPeer(void *pHead) {
+ if (tsSdbObj.sync == NULL) return TSDB_CODE_SUCCESS;
+
+ int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, NULL);
+ if (code > 0) {
+ sem_wait(&tsSdbObj.sem);
+ return tsSdbObj.code;
+ }
+ return code;
+}
+
+void sdbUpdateSync() {
+ SSyncCfg syncCfg = {0};
+ int32_t index = 0;
+
+ SDMNodeInfos *mnodes = dnodeGetMnodeList();
+ for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
+ SDMNodeInfo *node = &mnodes->nodeInfos[i];
+ syncCfg.nodeInfo[i].nodeId = node->nodeId;
+ syncCfg.nodeInfo[i].nodeIp = node->nodeIp;
+ strcpy(syncCfg.nodeInfo[i].name, node->nodeName);
+ index++;
+ }
+
+ if (index == 0) {
+ void *pNode = NULL;
+ while (1) {
+ SMnodeObj *pMnode = NULL;
+ pNode = mgmtGetNextMnode(pNode, &pMnode);
+ if (pMnode == NULL) break;
+
+ syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
+ syncCfg.nodeInfo[index].nodeIp = pMnode->pDnode->privateIp;
+ strcpy(syncCfg.nodeInfo[index].name, pMnode->pDnode->dnodeName);
+ index++;
+
+ mgmtReleaseMnode(pMnode);
+ }
+ }
+
+ syncCfg.replica = index;
+ syncCfg.arbitratorIp = syncCfg.nodeInfo[0].nodeIp;
+ if (syncCfg.replica == 1) {
+ syncCfg.quorum = 1;
+ } else {
+ syncCfg.quorum = 2;
+ }
+
+ bool hasThisDnode = false;
+ for (int32_t i = 0; i < syncCfg.replica; ++i) {
+ if (syncCfg.nodeInfo[i].nodeId == dnodeGetDnodeId()) {
+ hasThisDnode = true;
+ break;
+ }
+ }
+
+ if (!hasThisDnode) return;
+ if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
+
+ mPrint("work as mnode, replica:%d arbitratorIp:%s", syncCfg.replica, taosIpStr(syncCfg.arbitratorIp));
+ for (int32_t i = 0; i < syncCfg.replica; ++i) {
+ mPrint("mnode:%d, ip:%s name:%s", syncCfg.nodeInfo[i].nodeId, taosIpStr(syncCfg.nodeInfo[i].nodeIp),
+ syncCfg.nodeInfo[i].name);
}
- sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables);
+ SSyncInfo syncInfo;
+ syncInfo.vgId = 1;
+ syncInfo.version = sdbGetVersion();
+ syncInfo.syncCfg = syncCfg;
+ sprintf(syncInfo.path, "%s/", tsMnodeDir);
+ syncInfo.ahandle = NULL;
+ syncInfo.getWalInfo = sdbGetWalInfo;
+ syncInfo.getFileInfo = sdbGetFileInfo;
+ syncInfo.writeToCache = sdbWrite;
+ syncInfo.confirmForward = sdbConfirmForward;
+ syncInfo.notifyRole = sdbNotifyRole;
+ tsSdbObj.cfg = syncCfg;
+
+ if (tsSdbObj.sync) {
+ syncReconfig(tsSdbObj.sync, &syncCfg);
+ } else {
+ tsSdbObj.sync = syncStart(&syncInfo);
+ }
+}
+
+int32_t sdbInit() {
+ pthread_mutex_init(&tsSdbObj.mutex, NULL);
+ sem_init(&tsSdbObj.sem, 0, 0);
+
+ if (sdbInitWal() != 0) {
+ return -1;
+ }
- replicaNotify();
+ sdbRestoreTables();
+ if (mgmtGetMnodesNum() == 1) {
+ tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
+ }
+
+ sdbUpdateSync();
+
+ tsSdbObj.status = SDB_STATUS_SERVING;
return TSDB_CODE_SUCCESS;
}
void sdbCleanUp() {
- if (tsSdbObj) {
- pthread_mutex_destroy(&tsSdbObj->mutex);
- walClose(tsSdbObj->wal);
- free(tsSdbObj);
- tsSdbObj = NULL;
- }
+ if (tsSdbObj.status != SDB_STATUS_SERVING) return;
+
+ syncStop(tsSdbObj.sync);
+ free(tsSdbObj.sync);
+ walClose(tsSdbObj.wal);
+ sem_destroy(&tsSdbObj.sem);
+ pthread_mutex_destroy(&tsSdbObj.mutex);
+ memset(&tsSdbObj, 0, sizeof(tsSdbObj));
}
void sdbIncRef(void *handle, void *pRow) {
@@ -178,15 +353,15 @@ void sdbDecRef(void *handle, void *pRow) {
if (refCount <= 0 && *updateEnd) {
sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName,
sdbGetkeyStr(pTable, pRow), *pRefCount);
- SSdbOperDesc oper = {.pObj = pRow};
+ SSdbOper oper = {.pObj = pRow};
(*pTable->destroyFp)(&oper);
}
}
}
-static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
+static SSdbRow *sdbGetRowMeta(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
- SRowMeta * pMeta;
+ SSdbRow * pMeta;
if (handle == NULL) return NULL;
@@ -197,7 +372,7 @@ static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
- SRowMeta * pMeta;
+ SSdbRow * pMeta;
if (handle == NULL) return NULL;
@@ -213,8 +388,8 @@ void *sdbGetRow(void *handle, void *key) {
return pMeta->row;
}
-static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
- SRowMeta rowMeta;
+static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
+ SSdbRow rowMeta;
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
@@ -229,20 +404,20 @@ static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
pthread_mutex_unlock(&pTable->mutex);
- sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
+ sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->insertFp)(pOper);
return TSDB_CODE_SUCCESS;
}
-static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
+static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
pthread_mutex_lock(&pTable->mutex);
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
- sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
+ sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->deleteFp)(pOper);
@@ -253,127 +428,76 @@ static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
- sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
+static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
+ sdbTrace("table:%s, update record:%s in hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->updateFp)(pOper);
return TSDB_CODE_SUCCESS;
}
-static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
- int32_t code = 0;
+static int sdbWrite(void *param, void *data, int type) {
+ SWalHead *pHead = data;
+ int32_t tableId = pHead->msgType / 10;
+ int32_t action = pHead->msgType % 10;
- pthread_mutex_lock(&tsSdbObj->mutex);
- tsSdbObj->version++;
- pHead->version = tsSdbObj->version;
+ SSdbTable *pTable = sdbGetTableFromId(tableId);
+ assert(pTable != NULL);
- code = replicaForwardReqToPeer(pHead);
- if (code != TSDB_CODE_SUCCESS) {
- pthread_mutex_unlock(&tsSdbObj->mutex);
- sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName,
- sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
- return code;
+ pthread_mutex_lock(&tsSdbObj.mutex);
+ if (pHead->version == 0) {
+ // assign version
+ tsSdbObj.version++;
+ pHead->version = tsSdbObj.version;
+ } else {
+ // for data from WAL or forward, version may be smaller
+ if (pHead->version <= tsSdbObj.version) {
+ pthread_mutex_unlock(&tsSdbObj.mutex);
+ return TSDB_CODE_SUCCESS;
+ } else if (pHead->version != tsSdbObj.version + 1) {
+ pthread_mutex_unlock(&tsSdbObj.mutex);
+ sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64,
+ pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
+ tsSdbObj.version);
+ return TSDB_CODE_OTHERS;
+ } else {
+ tsSdbObj.version = pHead->version;
+ }
}
- code = walWrite(tsSdbObj->wal, pHead);
- pthread_mutex_unlock(&tsSdbObj->mutex);
-
+ int32_t code = walWrite(tsSdbObj.wal, pHead);
if (code < 0) {
- sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName,
- sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
- } else {
- sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action),
- sdbGetkeyStr(pTable, pHead->cont), pHead->version);
+ pthread_mutex_unlock(&tsSdbObj.mutex);
+ return code;
}
+ walFsync(tsSdbObj.wal);
- walFsync(tsSdbObj->wal);
- taosFreeQitem(pHead);
- return code;
-}
-
-static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
- pthread_mutex_lock(&tsSdbObj->mutex);
- if (pHead->version <= tsSdbObj->version) {
- pthread_mutex_unlock(&tsSdbObj->mutex);
- return TSDB_CODE_SUCCESS;
- } else if (pHead->version != tsSdbObj->version + 1) {
- pthread_mutex_unlock(&tsSdbObj->mutex);
- sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64,
- pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
- tsSdbObj->version);
- return TSDB_CODE_OTHERS;
- }
+ sdbForwardToPeer(pHead);
+ pthread_mutex_unlock(&tsSdbObj.mutex);
- tsSdbObj->version = pHead->version;
- sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName,
- sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
+ // from app, oper is created
+ if (param == NULL) return code;
- int32_t code = -1;
+ // from wal, should create oper
if (action == SDB_ACTION_INSERT) {
- SSdbOperDesc oper = {
- .rowSize = pHead->len,
- .rowData = pHead->cont,
- .table = pTable,
- };
+ SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper);
- if (code < 0) {
- sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
- sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
- pthread_mutex_unlock(&tsSdbObj->mutex);
- return code;
- }
-
- code = sdbInsertLocal(pTable, &oper);
+ return sdbInsertHash(pTable, &oper);
} else if (action == SDB_ACTION_DELETE) {
- SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
+ SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
-
- SSdbOperDesc oper = {
- .table = pTable,
- .pObj = rowMeta->row,
- };
-
- code = sdbDeleteLocal(pTable, &oper);
+ SSdbOper oper = {.table = pTable, .pObj = rowMeta->row};
+ return sdbDeleteHash(pTable, &oper);
} else if (action == SDB_ACTION_UPDATE) {
- SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
+ SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
-
- SSdbOperDesc oper = {
- .rowSize = pHead->len,
- .rowData = pHead->cont,
- .table = pTable,
- };
+ SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper);
- if (code < 0) {
- sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
- sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
- pthread_mutex_unlock(&tsSdbObj->mutex);
- return code;
- }
- code = sdbUpdateLocal(pTable, &oper);
- }
-
- pthread_mutex_unlock(&tsSdbObj->mutex);
- return code;
+ return sdbUpdateHash(pTable, &oper);
+ } else { return TSDB_CODE_INVALID_MSG_TYPE; }
}
-int sdbProcessWrite(void *param, void *data, int type) {
- SWalHead *pHead = data;
- int32_t tableId = pHead->msgType / 10;
- int32_t action = pHead->msgType % 10;
-
- SSdbTable *pTable = sdbGetTableFromId(tableId);
- assert(pTable != NULL);
-
- if (pHead->version == 0) {
- return sdbProcessWriteFromApp(pTable, pHead, action);
- } else {
- return sdbProcessWriteFromWal(pTable, pHead, action);
- }
-}
-
-int32_t sdbInsertRow(SSdbOperDesc *pOper) {
+int32_t sdbInsertRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
@@ -405,19 +529,19 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) {
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
- int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
+ int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
+ taosFreeQitem(pHead);
if (code < 0) return code;
- }
-
- return sdbInsertLocal(pTable, pOper);
+ }
+
+ return sdbInsertHash(pTable, pOper);
}
-// row here can be object or null-terminated string
-int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
+int32_t sdbDeleteRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
- SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
+ SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
@@ -447,18 +571,19 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
memcpy(pHead->cont, pOper->pObj, rowSize);
- int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
+ int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
+ taosFreeQitem(pHead);
if (code < 0) return code;
- }
-
- return sdbDeleteLocal(pTable, pOper);
+ }
+
+ return sdbDeleteHash(pTable, pOper);
}
-int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
+int32_t sdbUpdateRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
- SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
+ SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
@@ -477,16 +602,17 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
- int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
+ int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
+ taosFreeQitem(pHead);
if (code < 0) return code;
- }
+ }
- return sdbUpdateLocal(pTable, pOper);
+ return sdbUpdateHash(pTable, pOper);
}
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
SSdbTable *pTable = (SSdbTable *)handle;
- SRowMeta * pMeta;
+ SSdbRow * pMeta;
*ppRow = NULL;
if (pTable == NULL) return NULL;
@@ -520,13 +646,13 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
pTable->restoredFp = pDesc->restoredFp;
if (sdbInitIndexFp[pTable->keyType] != NULL) {
- pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta));
+ pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SSdbRow));
}
pthread_mutex_init(&pTable->mutex, NULL);
- tsSdbNumOfTables++;
- tsSdbTableList[pTable->tableId] = pTable;
+ tsSdbObj.numOfTables++;
+ tsSdbObj.tableList[pTable->tableId] = pTable;
return pTable;
}
@@ -534,16 +660,16 @@ void sdbCloseTable(void *handle) {
SSdbTable *pTable = (SSdbTable *)handle;
if (pTable == NULL) return;
- tsSdbNumOfTables--;
- tsSdbTableList[pTable->tableId] = NULL;
+ tsSdbObj.numOfTables--;
+ tsSdbObj.tableList[pTable->tableId] = NULL;
void *pNode = NULL;
while (1) {
- SRowMeta *pMeta;
+ SSdbRow *pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.pObj = pMeta->row,
.table = pTable,
};
@@ -557,6 +683,7 @@ void sdbCloseTable(void *handle) {
pthread_mutex_destroy(&pTable->mutex);
- sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables);
+ sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables);
free(pTable);
}
+
diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c
index 1dec4bd0153ee27299e946da933460eb7c5d802f..04f771081be058dfa15174dd522d3ac608c11c21 100644
--- a/src/mnode/src/mgmtShell.c
+++ b/src/mnode/src/mgmtShell.c
@@ -144,7 +144,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return;
}
- if (!mgmtIsMaster()) {
+ if (!sdbIsMaster()) {
// rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect());
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER);
rpcFreeCont(rpcMsg->pCont);
diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c
index 3fb4272b7f36f61ee31a2bb9029a03c3ac39e995..383b3474102c89f208f5a2860733c4f1e58e299e 100644
--- a/src/mnode/src/mgmtTable.c
+++ b/src/mnode/src/mgmtTable.c
@@ -83,12 +83,12 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) {
tfree(pTable);
}
-static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionDestroy(SSdbOper *pOper) {
mgmtDestroyChildTable(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
SChildTableObj *pTable = pOper->pObj;
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
@@ -128,7 +128,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) {
SChildTableObj *pTable = pOper->pObj;
if (pTable->vgId == 0) {
return TSDB_CODE_INVALID_VGROUP_ID;
@@ -169,7 +169,7 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
SChildTableObj *pNew = pOper->pObj;
SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId);
if (pTable != pNew) {
@@ -186,7 +186,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionEncode(SSdbOper *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SChildTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
@@ -208,7 +208,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
if (pTable == NULL) {
@@ -252,7 +252,7 @@ static int32_t mgmtChildTableActionRestored() {
SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
if (pDb == NULL) {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -266,7 +266,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pVgroup == NULL) {
mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -280,7 +280,7 @@ static int32_t mgmtChildTableActionRestored() {
mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -292,7 +292,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pVgroup->tableList == NULL) {
mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -306,7 +306,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pSuperTable == NULL) {
mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId);
pTable->vgId = 0;
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -347,7 +347,7 @@ static int32_t mgmtInitChildTables() {
return -1;
}
- mTrace("child table is initialized");
+ mTrace("table:ctables is created");
return 0;
}
@@ -392,12 +392,12 @@ static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
tfree(pStable);
}
-static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionDestroy(SSdbOper *pOper) {
mgmtDestroySuperTable(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionInsert(SSdbOper *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
@@ -408,7 +408,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionDelete(SSdbOper *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
@@ -420,7 +420,7 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
SSuperTableObj *pNew = pOper->pObj;
SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId);
if (pTable != pNew) {
@@ -435,7 +435,7 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionEncode(SSdbOper *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SSuperTableObj *pStable = pOper->pObj;
@@ -454,7 +454,7 @@ static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtSuperTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
@@ -505,7 +505,7 @@ static int32_t mgmtInitSuperTables() {
return -1;
}
- mTrace("stables is initialized");
+ mTrace("table:stables is created");
return 0;
}
@@ -731,7 +731,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
tschema[col].bytes = htons(tschema[col].bytes);
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -755,7 +755,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable
@@ -806,7 +806,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
pStable->numOfColumns += ntags;
pStable->sversion++;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -837,7 +837,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize);
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -872,7 +872,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema));
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -931,7 +931,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
mgmtDecAcctRef(pAcct);
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -968,7 +968,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
mgmtDecAcctRef(pAcct);
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@@ -1116,7 +1116,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
}
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsSuperTableSdb,
.pObj = pTable,
@@ -1354,7 +1354,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
}
}
- SSdbOperDesc desc = {0};
+ SSdbOper desc = {0};
desc.type = SDB_OPER_GLOBAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@@ -1508,7 +1508,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
mgmtDecAcctRef(pAcct);
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
@@ -1542,7 +1542,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
mgmtDecAcctRef(pAcct);
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
@@ -1687,7 +1687,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
}
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
@@ -1716,7 +1716,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
}
if (pTable->superTable == pStable) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
@@ -1805,7 +1805,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
return;
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
@@ -1848,7 +1848,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId,
queueMsg->thandle, tstrerror(rpcMsg->code));
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c
index 9098a0c17d4db0c8b955cc5b97c187f1d56262d5..dc4c3d6bea1b1193ec7fb9d05868ce7df23b2051 100644
--- a/src/mnode/src/mgmtUser.c
+++ b/src/mnode/src/mgmtUser.c
@@ -36,12 +36,12 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg);
-static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionInsert(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
@@ -56,7 +56,7 @@ static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionDelete(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
@@ -67,7 +67,7 @@ static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionUpdate(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SUserObj *pSaved = mgmtGetUser(pUser->user);
if (pUser != pSaved) {
@@ -77,14 +77,14 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionEncode(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtUserActionDecode(SSdbOper *pOper) {
SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj));
if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -137,7 +137,7 @@ int32_t mgmtInitUsers() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_USER, mgmtGetUserMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mgmtRetrieveUsers);
- mTrace("user data is initialized");
+ mTrace("table:users table is created");
return 0;
}
@@ -154,7 +154,7 @@ void mgmtReleaseUser(SUserObj *pUser) {
}
static int32_t mgmtUpdateUser(SUserObj *pUser) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
@@ -202,7 +202,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
pUser->superAuth = 1;
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
@@ -219,7 +219,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
}
static int32_t mgmtDropUser(SUserObj *pUser) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser
@@ -493,7 +493,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) {
if (pUser == NULL) break;
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsUserSdb,
.pObj = pUser,
diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c
index 9be95f4087aba02e4394793e9e7a1e4c24040ca4..87d8ddb2b4da298c57f0a369fe5c9e3f69aea692 100644
--- a/src/mnode/src/mgmtVgroup.c
+++ b/src/mnode/src/mgmtVgroup.c
@@ -22,7 +22,7 @@
#include "tidpool.h"
#include "tsync.h"
#include "ttime.h"
-#include "treplica.h"
+#include "tbalance.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtDb.h"
@@ -48,7 +48,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ;
static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle);
static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
-static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionDestroy(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->idPool) {
taosIdPoolCleanUp(pVgroup->idPool);
@@ -62,7 +62,7 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
@@ -104,7 +104,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->pDb != NULL) {
@@ -124,7 +124,7 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
SVgObj *pNew = pOper->pObj;
SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
if (pVgroup != pNew) {
@@ -147,14 +147,14 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionEncode(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
-static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
+static int32_t mgmtVgroupActionDecode(SSdbOper *pOper) {
SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj));
if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@@ -199,7 +199,7 @@ int32_t mgmtInitVgroups() {
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp);
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
- mTrace("vgroup is initialized");
+ mTrace("table:vgroups is created");
return 0;
}
@@ -213,7 +213,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
}
void mgmtUpdateVgroup(SVgObj *pVgroup) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@@ -249,7 +249,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
strcpy(pVgroup->dbName, pDb->name);
pVgroup->numOfVnodes = pDb->cfg.replications;
pVgroup->createdTime = taosGetTimestampMs();
- if (replicaAllocVnodes(pVgroup) != 0) {
+ if (balanceAllocVnodes(pVgroup) != 0) {
mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
free(pVgroup);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
@@ -257,7 +257,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
return;
}
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@@ -289,7 +289,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
} else {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@@ -596,7 +596,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(queueMsg);
mgmtAddToShellQueue(newMsg);
} else {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@@ -659,7 +659,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
if (queueMsg->received != queueMsg->expected) return;
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@@ -716,7 +716,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
if (pVgroup == NULL) break;
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
- SSdbOperDesc oper = {
+ SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c
index 8ee337b8edd240d101601ceac38c5f33f2852411..b24e5b2a4bbf35072cf023fff411563bd7effd55 100644
--- a/src/query/src/queryExecutor.c
+++ b/src/query/src/queryExecutor.c
@@ -2479,7 +2479,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d",
GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order);
- tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle;
+ TsdbQueryHandleT pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle;
while (tsdbNextDataBlock(pQueryHandle)) {
if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
@@ -4221,7 +4221,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) {
int64_t st = taosGetTimestampMs();
- tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle;
+ TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle;
while (tsdbNextDataBlock(pQueryHandle)) {
if (isQueryKilled(pQInfo)) {
break;
diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h
index 37963a33226813816893c7a855c74b63325ddf6e..bcf282ba511fac7f2e06ad7a3744dc3039922b17 100644
--- a/src/tsdb/inc/tsdbMain.h
+++ b/src/tsdb/inc/tsdbMain.h
@@ -120,7 +120,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable);
#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
#define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
-STsdbMeta *tsdbGetMeta(tsdb_repo_t *pRepo);
+STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo);
int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
@@ -160,10 +160,10 @@ typedef struct {
STsdbCacheBlock *curBlock;
SCacheMem * mem;
SCacheMem * imem;
- tsdb_repo_t * pRepo;
+ TsdbRepoT * pRepo;
} STsdbCache;
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo);
+STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo);
void tsdbFreeCache(STsdbCache *pCache);
void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
@@ -220,11 +220,12 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles);
void tsdbCloseFileH(STsdbFileH *pFileH);
int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader,
int toClose);
-int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
+SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
int tsdbOpenFile(SFile *pFile, int oflag);
int tsdbCloseFile(SFile *pFile);
SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid);
int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid);
+int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
@@ -270,6 +271,8 @@ typedef struct {
TSKEY keyLast;
} SCompBlock;
+// Maximum number of sub-blocks a super-block can have
+#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1)
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
@@ -307,19 +310,11 @@ typedef struct {
SCompCol cols[];
} SCompData;
-STsdbFileH *tsdbGetFile(tsdb_repo_t *pRepo);
-
-int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast,
- SDataCols *pCols);
-
-int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables);
-int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf);
-int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf);
-int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf);
-int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData);
+STsdbFileH *tsdbGetFile(TsdbRepoT *pRepo);
+int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast,
+ SDataCols *pCols);
SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid);
-
void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey);
// TSDB repository definition
@@ -375,9 +370,103 @@ typedef struct {
int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
-int32_t tsdbTriggerCommit(tsdb_repo_t *repo);
-int32_t tsdbLockRepo(tsdb_repo_t *repo);
-int32_t tsdbUnLockRepo(tsdb_repo_t *repo);
+int32_t tsdbTriggerCommit(TsdbRepoT *repo);
+int32_t tsdbLockRepo(TsdbRepoT *repo);
+int32_t tsdbUnLockRepo(TsdbRepoT *repo);
+
+typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
+
+typedef struct {
+ tsdb_rw_helper_t type; // helper type
+
+ int maxTables;
+ int maxRowSize;
+ int maxRows;
+ int maxCols;
+ int minRowsPerFileBlock;
+ int maxRowsPerFileBlock;
+ int8_t compress;
+} SHelperCfg;
+
+typedef struct {
+ int fid;
+ TSKEY minKey;
+ TSKEY maxKey;
+ // For read/write purpose
+ SFile headF;
+ SFile dataF;
+ SFile lastF;
+ // For write purpose only
+ SFile nHeadF;
+ SFile nLastF;
+} SHelperFile;
+
+typedef struct {
+ int64_t uid;
+ int32_t tid;
+ int32_t sversion;
+} SHelperTable;
+
+typedef struct {
+ // Global configuration
+ SHelperCfg config;
+
+ int8_t state;
+
+ // For file set usage
+ SHelperFile files;
+ SCompIdx * pCompIdx;
+
+ // For table set usage
+ SHelperTable tableInfo;
+ SCompInfo * pCompInfo;
+ bool hasOldLastBlock;
+
+ // For block set usage
+ SCompData *pCompData;
+ SDataCols *pDataCols[2];
+
+} SRWHelper;
+
+// --------- Helper state
+#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state
+#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set
+#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded
+#define TSDB_HELPER_TABLE_SET 0x4 // Table is set
+#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded
+#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded
+
+#define TSDB_HELPER_TYPE(h) ((h)->config.type)
+
+#define helperSetState(h, s) (((h)->state) |= (s))
+#define helperClearState(h, s) ((h)->state &= (~(s)))
+#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
+#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
+
+int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
+int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
+// int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
+void tsdbDestroyHelper(SRWHelper *pHelper);
+void tsdbResetHelper(SRWHelper *pHelper);
+
+// --------- For set operations
+int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
+// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
+void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
+int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
+
+// --------- For read operations
+int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
+int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
+int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
+int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
+int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
+
+// --------- For write operations
+int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);
+int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper);
+int tsdbWriteCompInfo(SRWHelper *pHelper);
+int tsdbWriteCompIdx(SRWHelper *pHelper);
#ifdef __cplusplus
}
diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c
index 324991a04672474adb9a444b282c638eccafbce2..942afd70a6875215686dcbe67d3cc551fc9b60ba 100644
--- a/src/tsdb/src/tsdbCache.c
+++ b/src/tsdb/src/tsdbCache.c
@@ -21,7 +21,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache);
static void tsdbFreeBlockList(SList *list);
static void tsdbFreeCacheMem(SCacheMem *mem);
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo) {
+STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
if (pCache == NULL) return NULL;
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index 7576717dd1a9eeef30d3ef9aa7eaee6373d74fff..ab76f69bedf1d289be57f7b2539ff1b9fbcac466 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -21,10 +21,12 @@
#include
#include
#include
+#include
+#include "talgo.h"
+#include "tchecksum.h"
#include "tsdbMain.h"
#include "tutil.h"
-#include "talgo.h"
const char *tsdbFileSuffix[] = {
".head", // TSDB_FILE_TYPE_HEAD
@@ -34,7 +36,6 @@ const char *tsdbFileSuffix[] = {
static int compFGroupKey(const void *key, const void *fgroup);
static int compFGroup(const void *arg1, const void *arg2);
-static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
static int tsdbWriteFileHead(SFile *pFile);
static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables);
static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
@@ -93,24 +94,36 @@ static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) {
return 0;
}
-int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) {
- if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1;
+/**
+ * Create the file group if the file group not exists.
+ *
+ * @return A pointer to
+ */
+SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) {
+ if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL;
SFileGroup fGroup;
SFileGroup *pFGroup = &fGroup;
- if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one
+
+ SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid);
+ if (pGroup == NULL) { // if not exists, create one
pFGroup->fileId = fid;
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) {
- // TODO: deal with the ERROR here, remove those creaed file
- return -1;
- }
+ if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]),
+ type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0)
+ goto _err;
}
pFileH->fGroup[pFileH->numOfFGroups++] = fGroup;
qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup);
+ return tsdbSearchFGroup(pFileH, fid);
}
- return 0;
+
+ return pGroup;
+
+_err:
+ // TODO: deal with the err here
+ return NULL;
}
int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) {
@@ -183,27 +196,27 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
return ret;
}
-int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
- SCompBlock *pBlock = pStartBlock;
- for (int i = 0; i < numOfBlocks; i++) {
- if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
- pCols->numOfPoints += (pCompData->cols[0].len / 8);
- for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
- SCompCol *pCompCol = &(pCompData->cols[iCol]);
- // pCols->numOfPoints += pBlock->numOfPoints;
- int k = 0;
- for (; k < pCols->numOfCols; k++) {
- if (pCompCol->colId == pCols->cols[k].colId) break;
- }
-
- if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
- (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
- return -1;
- }
- pStartBlock++;
- }
- return 0;
-}
+// int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
+// SCompBlock *pBlock = pStartBlock;
+// for (int i = 0; i < numOfBlocks; i++) {
+// if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
+// pCols->numOfPoints += (pCompData->cols[0].len / 8);
+// for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
+// SCompCol *pCompCol = &(pCompData->cols[iCol]);
+// // pCols->numOfPoints += pBlock->numOfPoints;
+// int k = 0;
+// for (; k < pCols->numOfCols; k++) {
+// if (pCompCol->colId == pCols->cols[k].colId) break;
+// }
+
+// if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
+// (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
+// return -1;
+// }
+// pStartBlock++;
+// }
+// return 0;
+// }
int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) {
SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx);
@@ -239,42 +252,42 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf
return 0;
}
-int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) {
- SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
- if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+// int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) {
+// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
+// if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
- if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1;
- // TODO: need to check the correctness
- return 0;
-}
+// if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1;
+// // TODO: need to check the correctness
+// return 0;
+// }
-int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) {
- SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
+// int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) {
+// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
- if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1;
+// if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1;
- if (read(pFile->fd, buf, pIdx->len) < 0) return -1;
+// if (read(pFile->fd, buf, pIdx->len) < 0) return -1;
- // TODO: need to check the correctness
+// // TODO: need to check the correctness
- return 0;
-}
+// return 0;
+// }
-int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) {
- // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+// int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) {
+// // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
- if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1;
- size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols;
- if (read(pFile->fd, buf, size) < 0) return -1;
+// if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1;
+// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols;
+// if (read(pFile->fd, buf, size) < 0) return -1;
- return 0;
-}
+// return 0;
+// }
-int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) {
- if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1;
- if (read(pFile->fd, buf, pCol->len) < 0) return -1;
- return 0;
-}
+// int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) {
+// if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1;
+// if (read(pFile->fd, buf, pCol->len) < 0) return -1;
+// return 0;
+// }
static int compFGroupKey(const void *key, const void *fgroup) {
int fid = *(int *)key;
@@ -299,7 +312,7 @@ static int tsdbWriteFileHead(SFile *pFile) {
}
static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
- int size = sizeof(SCompIdx) * maxTables;
+ int size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM);
void *buf = calloc(1, size);
if (buf == NULL) return -1;
@@ -308,6 +321,8 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return -1;
}
+ taosCalcChecksumAppend(0, (uint8_t *)buf, size);
+
if (write(pFile->fd, buf, size) < 0) {
free(buf);
return -1;
@@ -319,7 +334,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return 0;
}
-static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
+int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
if (dataDir == NULL || fname == NULL) return -1;
sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix);
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index 9c0050b38fc2f331ff7ca754cc6d86656b4166d5..13dbf7eb8cab5f480ff77b6aafe524ef9d8e98f8 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -12,15 +12,16 @@
#include
#include
-// #include "taosdef.h"
-// #include "disk.h"
#include "os.h"
#include "talgo.h"
#include "tsdb.h"
#include "tsdbMain.h"
+#include "tscompression.h"
#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision
#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
+#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
+#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
#define TSDB_MIN_ID 0
#define TSDB_MAX_ID INT_MAX
#define TSDB_MIN_TABLES 10
@@ -53,15 +54,16 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
// static int tsdbOpenMetaFile(char *tsdbDir);
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock);
static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
static void * tsdbCommitData(void *arg);
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols);
-static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey);
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper,
+ SDataCols *pDataCols);
+static TSKEY tsdbNextIterKey(SSkipListIterator *pIter);
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
-static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
- int64_t uid);
+// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
+// int64_t uid);
#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@@ -82,6 +84,7 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
pCfg->maxRowsPerFileBlock = -1;
pCfg->keep = -1;
pCfg->maxCacheSize = -1;
+ pCfg->compression = TWO_STAGE_COMP;
}
/**
@@ -153,7 +156,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */)
*
* @return 0 for success, -1 for failure and the error number is set
*/
-int32_t tsdbDropRepo(tsdb_repo_t *repo) {
+int32_t tsdbDropRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
pRepo->state = TSDB_REPO_STATE_CLOSED;
@@ -179,7 +182,7 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) {
*
* @return a TSDB repository handle on success, NULL for failure and the error number is set
*/
-tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
+TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
char dataDir[128] = "\0";
if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) {
return NULL;
@@ -202,7 +205,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
return NULL;
}
- pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (tsdb_repo_t *)pRepo);
+ pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo);
if (pRepo->tsdbCache == NULL) {
tsdbFreeMeta(pRepo->tsdbMeta);
free(pRepo->rootDir);
@@ -222,7 +225,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
pRepo->state = TSDB_REPO_STATE_ACTIVE;
- return (tsdb_repo_t *)pRepo;
+ return (TsdbRepoT *)pRepo;
}
// static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
@@ -237,7 +240,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
*
* @return 0 for success, -1 for failure and the error number is set
*/
-int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
+int32_t tsdbCloseRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
if (pRepo == NULL) return 0;
@@ -282,7 +285,7 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
*
* @return 0 for success, -1 for failure and the error number is set
*/
-int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
+int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
pRepo->config = *pCfg;
@@ -290,7 +293,7 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
return 0;
}
-int32_t tsdbTriggerCommit(tsdb_repo_t *repo) {
+int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
tsdbLockRepo(repo);
@@ -322,12 +325,12 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) {
return 0;
}
-int32_t tsdbLockRepo(tsdb_repo_t *repo) {
+int32_t tsdbLockRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return pthread_mutex_lock(&(pRepo->mutex));
}
-int32_t tsdbUnLockRepo(tsdb_repo_t *repo) {
+int32_t tsdbUnLockRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return pthread_mutex_unlock(&(pRepo->mutex));
}
@@ -340,35 +343,35 @@ int32_t tsdbUnLockRepo(tsdb_repo_t *repo) {
* @return a info struct handle on success, NULL for failure and the error number is set. The upper
* layers should free the info handle themselves or memory leak will occur
*/
-STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
+STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) {
// TODO
return NULL;
}
-int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
+int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
}
-int tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
+int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
// TODO
return 0;
}
-int tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
+int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
if (repo == NULL) return -1;
STsdbRepo *pRepo = (STsdbRepo *)repo;
return tsdbDropTableImpl(pRepo->tsdbMeta, tableId);
}
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) {
+STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
// TODO
return NULL;
}
// TODO: need to return the number of data inserted
-int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
+int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) {
SSubmitMsgIter msgIter;
tsdbInitSubmitMsgIter(pMsg, &msgIter);
@@ -397,6 +400,7 @@ int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t ti
config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
config->tableId.uid = uid;
config->tableId.tid = tid;
+ config->name = strdup("test1");
return 0;
}
@@ -552,12 +556,12 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
return pBlock;
}
-STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) {
+STsdbMeta* tsdbGetMeta(TsdbRepoT* pRepo) {
STsdbRepo *tsdb = (STsdbRepo *)pRepo;
return tsdb->tsdbMeta;
}
-STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo) {
+STsdbFileH* tsdbGetFile(TsdbRepoT* pRepo) {
STsdbRepo* tsdb = (STsdbRepo*) pRepo;
return tsdb->tsdbFileH;
}
@@ -571,6 +575,13 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
if (!IS_VALID_PRECISION(pCfg->precision)) return -1;
}
+ // Check compression
+ if (pCfg->compression == -1) {
+ pCfg->compression = TSDB_DEFAULT_COMPRESSION;
+ } else {
+ if (!IS_VALID_COMPRESSION(pCfg->compression)) return -1;
+ }
+
// Check tsdbId
if (pCfg->tsdbId < 0) return -1;
@@ -761,7 +772,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
return 0;
}
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
+static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
@@ -785,6 +796,9 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
}
static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) {
+ ASSERT(maxRowsToRead > 0);
+ if (pIter == NULL) return 0;
+
int numOfRows = 0;
do {
@@ -823,19 +837,16 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
if (pTable == NULL || pTable->imem == NULL) continue;
iters[tid] = tSkipListCreateIter(pTable->imem->pData);
- if (iters[tid] == NULL) {
- tsdbDestroyTableIters(iters, maxTables);
- return NULL;
- }
+ if (iters[tid] == NULL) goto _err;
- if (!tSkipListIterNext(iters[tid])) {
- // No data in this iterator
- tSkipListDestroyIter(iters[tid]);
- iters[tid] = NULL;
- }
+ if (!tSkipListIterNext(iters[tid])) goto _err;
}
return iters;
+
+ _err:
+ tsdbDestroyTableIters(iters, maxTables);
+ return NULL;
}
static void tsdbFreeMemTable(SMemTable *pMemTable) {
@@ -847,14 +858,17 @@ static void tsdbFreeMemTable(SMemTable *pMemTable) {
// Commit to file
static void *tsdbCommitData(void *arg) {
- // TODO
printf("Starting to commit....\n");
STsdbRepo * pRepo = (STsdbRepo *)arg;
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbCache *pCache = pRepo->tsdbCache;
- STsdbCfg * pCfg = &(pRepo->config);
+ STsdbCfg * pCfg = &(pRepo->config);
+ SDataCols * pDataCols = NULL;
+ SRWHelper whelper = {0};
if (pCache->imem == NULL) return NULL;
+ pRepo->appH.walCallBack(pRepo->appH.appH);
+
// Create the iterator to read from cache
SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables);
if (iters == NULL) {
@@ -862,23 +876,21 @@ static void *tsdbCommitData(void *arg) {
return NULL;
}
- // Create a data column buffer for commit
- SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock);
- if (pDataCols == NULL) {
- // TODO: deal with the error
- return NULL;
- }
+ if (tsdbInitWriteHelper(&whelper, pRepo) < 0) goto _exit;
+ if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit;
int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision);
int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision);
+ // Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
- if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) {
- // TODO: deal with the error here
- // assert(0);
+ if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
+ ASSERT(false);
+ goto _exit;
}
}
+_exit:
tdFreeDataCols(pDataCols);
tsdbDestroyTableIters(iters, pCfg->maxTables);
@@ -888,7 +900,6 @@ static void *tsdbCommitData(void *arg) {
free(pCache->imem);
pCache->imem = NULL;
pRepo->commit = 0;
- // TODO: free the skiplist
for (int i = 0; i < pCfg->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->imem) {
@@ -901,19 +912,12 @@ static void *tsdbCommitData(void *arg) {
return NULL;
}
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) {
- int isNewLastFile = 0;
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
STsdbCfg * pCfg = &pRepo->config;
- SFile hFile, lFile;
SFileGroup *pGroup = NULL;
- SCompIdx * pIndices = NULL;
- SCompInfo * pCompInfo = NULL;
- // size_t compInfoSize = 0;
- // SCompBlock compBlock;
- // SCompBlock *pBlock = &compBlock;
TSKEY minKey = 0, maxKey = 0;
tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
@@ -922,334 +926,93 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey);
if (!hasDataToCommit) return 0; // No data to commit, just return
- // TODO: make it more flexible
- pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000);
-
// Create and open files for commit
tsdbGetDataDirName(pRepo, dataDir);
- if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */
- }
- pGroup = tsdbOpenFilesForCommit(pFileH, fid);
- if (pGroup == NULL) { /* TODO */
- }
- tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 1);
- tsdbOpenFile(&hFile, O_RDWR);
- if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) {
- // TODO: make it not to write the last file every time
- tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0);
- isNewLastFile = 1;
- }
+ if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err;
- // Load the SCompIdx
- pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables);
- if (pIndices == NULL) { /* TODO*/
- }
- if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */
- }
-
- lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET);
+ // Open files for write/read
+ if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
// Loop to commit data in each table
for (int tid = 0; tid < pCfg->maxTables; tid++) {
STable * pTable = pMeta->tables[tid];
- SSkipListIterator *pIter = iters[tid];
- SCompIdx * pIdx = &pIndices[tid];
-
- int nNewBlocks = 0;
-
- if (pTable == NULL || pIter == NULL) continue;
-
- /* If no new data to write for this table, just write the old data to new file
- * if there are.
- */
- if (!tsdbHasDataInRange(pIter, minKey, maxKey)) {
- // has old data
- if (pIdx->len > 0) {
- goto _table_over;
- // if (isNewLastFile && pIdx->hasLast) {
- if (0) {
- // need to move the last block to new file
- if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */
- }
- if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */
- }
-
- tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable));
-
- SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks);
- int nBlocks = 0;
-
- TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks);
-
- SCompData tBlock;
- int64_t toffset;
- int32_t tlen;
- tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock);
-
- tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid);
- pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks);
- pTBlock->offset = toffset;
- pTBlock->len = tlen;
- pTBlock->numOfPoints = pCols->numOfPoints;
- pTBlock->numOfSubBlocks = 1;
-
- pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR);
- if (nBlocks > 1) {
- pIdx->len -= (sizeof(SCompBlock) * nBlocks);
- }
- write(hFile.fd, (void *)pCompInfo, pIdx->len);
- } else {
- pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR);
- sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len);
- hFile.info.size += pIdx->len;
- }
- }
- continue;
- }
+ if (pTable == NULL) continue;
- pCompInfo->delimiter = TSDB_FILE_DELIMITER;
- pCompInfo->checksum = 0;
- pCompInfo->uid = pTable->tableId.uid;
-
- // Load SCompBlock part if neccessary
- // int isCompBlockLoaded = 0;
- if (0) {
- // if (pIdx->offset > 0) {
- if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) {
- // has last block || cache key overlap with commit key
- pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100);
- if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */
- }
- // if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1;
- } else {
- // TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part
- // and write those new blocks to it
- }
- }
+ SSkipListIterator *pIter = iters[tid];
- tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable));
+ // Set the helper and the buffer dataCols object to help to write this table
+ tsdbSetHelperTable(pHelper, pTable, pRepo);
+ tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable));
+ // Loop to write the data in the cache to files. If no data to write, just break the loop
int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5;
- while (1) {
- tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols);
- if (pCols->numOfPoints == 0) break;
-
- int pointsWritten = pCols->numOfPoints;
- // TODO: all write to the end of .data file
- int64_t toffset = 0;
- int32_t tlen = 0;
- tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid);
-
- // Make the compBlock
- SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++;
- pTBlock->offset = toffset;
- pTBlock->len = tlen;
- pTBlock->keyFirst = dataColsKeyFirst(pCols);
- pTBlock->keyLast = dataColsKeyLast(pCols);
- pTBlock->last = 0;
- pTBlock->algorithm = 0;
- pTBlock->numOfPoints = pCols->numOfPoints;
- pTBlock->sversion = pTable->sversion;
- pTBlock->numOfSubBlocks = 1;
- pTBlock->numOfCols = pCols->numOfCols;
-
- if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols);
-
- tdPopDataColsPoints(pCols, pointsWritten);
- maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints;
+ int nLoop = 0;
+ while (true) {
+ int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols);
+ assert(rowsRead >= 0);
+ if (pDataCols->numOfPoints == 0) break;
+ nLoop++;
+
+ ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
+ ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
+
+ int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
+ ASSERT(rowsWritten != 0);
+ if (rowsWritten < 0) goto _err;
+ ASSERT(rowsWritten <= pDataCols->numOfPoints);
+
+ tdPopDataColsPoints(pDataCols, rowsWritten);
+ maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfPoints;
}
+ ASSERT(pDataCols->numOfPoints == 0);
-_table_over:
- // Write the SCompBlock part
- pIdx->offset = lseek(hFile.fd, 0, SEEK_END);
- if (pIdx->len > 0) {
- int bytes = tsendfile(hFile.fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len);
- if (bytes < pIdx->len) {
- printf("Failed to send file, reason: %s\n", strerror(errno));
- }
- if (nNewBlocks > 0) {
- write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks);
- pIdx->len += (sizeof(SCompBlock) * nNewBlocks);
- }
- } else {
- if (nNewBlocks > 0) {
- write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks);
- pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks;
- }
- }
+ // Move the last block to the new .l file if neccessary
+ if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err;
- pIdx->checksum = 0;
- pIdx->numOfSuperBlocks += nNewBlocks;
- pIdx->hasLast = 0;
+ // Write the SCompBlock part
+ if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
+
}
- // Write the SCompIdx part
- if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */}
- if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */}
-
- // close the files
- for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- tsdbCloseFile(&pGroup->files[type]);
- }
- tsdbCloseFile(&hFile);
- if (isNewLastFile) tsdbCloseFile(&lFile);
- // TODO: replace the .head and .last file
- rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname);
- pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info;
- if (isNewLastFile) {
- rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname);
- pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info;
- }
+ if (tsdbWriteCompIdx(pHelper) < 0) goto _err;
- if (pIndices) free(pIndices);
- if (pCompInfo) free(pCompInfo);
+ tsdbCloseHelperFile(pHelper, 0);
+ // TODO: make it atomic with some methods
+ pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF;
+ pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF;
+ pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF;
return 0;
+
+ _err:
+ ASSERT(false);
+ tsdbCloseHelperFile(pHelper, 1);
+ return -1;
}
-static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) {
- if (pIter == NULL) return 0;
+/**
+ * Return the next iterator key.
+ *
+ * @return the next key if iter has
+ * -1 if iter not
+ */
+static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
+ if (pIter == NULL) return -1;
SSkipListNode *node = tSkipListIterGet(pIter);
- if (node == NULL) return 0;
+ if (node == NULL) return -1;
SDataRow row = SL_GET_NODE_DATA(node);
- if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1;
-
- return 0;
+ return dataRowKey(row);
}
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) {
+ TSKEY nextKey;
for (int i = 0; i < nIters; i++) {
SSkipListIterator *pIter = iters[i];
- if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1;
- }
- return 0;
-}
-
-static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) {
- size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols;
- SCompData *pCompData = (SCompData *)malloc(size);
- if (pCompData == NULL) return -1;
-
- pCompData->delimiter = TSDB_FILE_DELIMITER;
- pCompData->uid = uid;
- pCompData->numOfCols = pCols->numOfCols;
-
- *offset = lseek(pFile->fd, 0, SEEK_END);
- *len = size;
-
- int toffset = size;
- for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
- SCompCol *pCompCol = pCompData->cols + iCol;
- SDataCol *pDataCol = pCols->cols + iCol;
-
- pCompCol->colId = pDataCol->colId;
- pCompCol->type = pDataCol->type;
- pCompCol->offset = toffset;
-
- // TODO: add compression
- pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite;
- toffset += pCompCol->len;
- }
-
- // Write the block
- if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err;
- *len += size;
- for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
- SDataCol *pDataCol = pCols->cols + iCol;
- SCompCol *pCompCol = pCompData->cols + iCol;
- if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err;
- *len += pCompCol->len;
+ nextKey = tsdbNextIterKey(pIter);
+ if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
-
- tfree(pCompData);
return 0;
-
-_err:
- tfree(pCompData);
- return -1;
-}
-
-static int compareKeyBlock(const void *arg1, const void *arg2) {
- TSKEY key = *(TSKEY *)arg1;
- SCompBlock *pBlock = (SCompBlock *)arg2;
-
- if (key < pBlock->keyFirst) {
- return -1;
- } else if (key > pBlock->keyLast) {
- return 1;
- }
-
- return 0;
-}
-
-int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) {
- STsdbCfg * pCfg = &(pRepo->config);
- SFile * pFile = NULL;
- int numOfPointsToWrite = 0;
- int64_t offset = 0;
- int32_t len = 0;
-
- memset((void *)pCompBlock, 0, sizeof(SCompBlock));
-
- if (pCompInfo == NULL) {
- // Just append the data block to .data or .l or .last file
- numOfPointsToWrite = pCols->numOfPoints;
- if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file
- pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]);
- } else { // Write to .last or .l file
- pCompBlock->last = 1;
- if (lFile) {
- pFile = lFile;
- } else {
- pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]);
- }
- }
- tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid);
- pCompBlock->offset = offset;
- pCompBlock->len = len;
- pCompBlock->algorithm = 2; // TODO : add to configuration
- pCompBlock->sversion = pCols->sversion;
- pCompBlock->numOfPoints = pCols->numOfPoints;
- pCompBlock->numOfSubBlocks = 1;
- pCompBlock->numOfCols = pCols->numOfCols;
- pCompBlock->keyFirst = dataColsKeyFirst(pCols);
- pCompBlock->keyLast = dataColsKeyLast(pCols);
- } else {
- // Need to merge the block to either the last block or the other block
- TSKEY keyFirst = dataColsKeyFirst(pCols);
- SCompBlock *pMergeBlock = NULL;
-
- // Search the block to merge in
- void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks,
- compareKeyBlock, TD_GE);
- if (ptr == NULL) {
- // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block
- // and merge the data
- pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1);
- } else {
- pMergeBlock = (SCompBlock *)ptr;
- }
-
- if (pMergeBlock->last) {
- if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) {
- // Need to load the data from .last and combine data in pCols to write to .data file
-
- } else { // Just append the block to .last or .l file
- if (lFile) {
- // read the block from .last file and merge with pCols, write to .l file
-
- } else {
- // tsdbWriteBlockToFileImpl();
- }
- }
- } else { // The block need to merge in .data file
-
- }
-
- }
-
- return numOfPointsToWrite;
-}
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index 716f888153d8b942d44d47c4fce3fcb5568b28cb..7c8d1fd51ffd1bd7bbcbb020ae1c67bdea371c28 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -225,7 +225,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) {
}
}
-int32_t tsdbGetTableTagVal(tsdb_repo_t* repo, STableId id, int32_t colId, int16_t* type, int16_t* bytes, char** val) {
+int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t* type, int16_t* bytes, char** val) {
STsdbMeta* pMeta = tsdbGetMeta(repo);
STable* pTable = tsdbGetTableByUid(pMeta, id.uid);
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
new file mode 100644
index 0000000000000000000000000000000000000000..1b5b0e765691a95040925f142f7b8b0cd2bc0c7d
--- /dev/null
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -0,0 +1,1128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "tsdbMain.h"
+#include "tchecksum.h"
+#include "tscompression.h"
+#include "talgo.h"
+
+// Local function definitions
+// static int tsdbCheckHelperCfg(SHelperCfg *pCfg);
+static int tsdbInitHelperFile(SRWHelper *pHelper);
+// static void tsdbClearHelperFile(SHelperFile *pHFile);
+static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
+static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite,
+ SCompBlock *pCompBlock, bool isLast, bool isSuperBlock);
+static int compareKeyBlock(const void *arg1, const void *arg2);
+static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols);
+static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
+static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
+static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey);
+static void tsdbResetHelperBlock(SRWHelper *pHelper);
+
+// ---------- Operations on Helper File part
+static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
+ memset((void *)&pHelper->files, 0, sizeof(pHelper->files));
+ pHelper->files.fid = -1;
+ pHelper->files.headF.fd = -1;
+ pHelper->files.dataF.fd = -1;
+ pHelper->files.lastF.fd = -1;
+ pHelper->files.nHeadF.fd = -1;
+ pHelper->files.nLastF.fd = -1;
+}
+
+static int tsdbInitHelperFile(SRWHelper *pHelper) {
+ // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
+ size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
+ pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize);
+ if (pHelper->pCompIdx == NULL) return -1;
+
+ tsdbResetHelperFileImpl(pHelper);
+ return 0;
+}
+
+static void tsdbDestroyHelperFile(SRWHelper *pHelper) {
+ tsdbCloseHelperFile(pHelper, false);
+ tzfree(pHelper->pCompIdx);
+}
+
+// ---------- Operations on Helper Table part
+static void tsdbResetHelperTableImpl(SRWHelper *pHelper) {
+ memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable));
+ pHelper->hasOldLastBlock = false;
+}
+
+static void tsdbResetHelperTable(SRWHelper *pHelper) {
+ tsdbResetHelperBlock(pHelper);
+ tsdbResetHelperTableImpl(pHelper);
+ helperClearState(pHelper, (TSDB_HELPER_TABLE_SET|TSDB_HELPER_INFO_LOAD));
+}
+
+static void tsdbInitHelperTable(SRWHelper *pHelper) {
+ tsdbResetHelperTableImpl(pHelper);
+}
+
+static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); }
+
+// ---------- Operations on Helper Block part
+static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) {
+ tdResetDataCols(pHelper->pDataCols[0]);
+ tdResetDataCols(pHelper->pDataCols[1]);
+}
+
+static void tsdbResetHelperBlock(SRWHelper *pHelper) {
+ tsdbResetHelperBlockImpl(pHelper);
+ // helperClearState(pHelper, TSDB_HELPER_)
+}
+
+static int tsdbInitHelperBlock(SRWHelper *pHelper) {
+ pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows);
+ pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows);
+ if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1;
+
+ tsdbResetHelperBlockImpl(pHelper);
+
+ return 0;
+}
+
+static void tsdbDestroyHelperBlock(SRWHelper *pHelper) {
+ tzfree(pHelper->pCompData);
+ tdFreeDataCols(pHelper->pDataCols[0]);
+ tdFreeDataCols(pHelper->pDataCols[1]);
+}
+
+static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) {
+ if (pHelper == NULL || pRepo == NULL) return -1;
+
+ memset((void *)pHelper, 0, sizeof(*pHelper));
+
+ // Init global configuration
+ pHelper->config.type = type;
+ pHelper->config.maxTables = pRepo->config.maxTables;
+ pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes;
+ pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock;
+ pHelper->config.maxCols = pRepo->tsdbMeta->maxCols;
+ pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock;
+ pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock;
+ pHelper->config.compress = pRepo->config.compression;
+
+ pHelper->state = TSDB_HELPER_CLEAR_STATE;
+
+ // Init file part
+ if (tsdbInitHelperFile(pHelper) < 0) goto _err;
+
+ // Init table part
+ tsdbInitHelperTable(pHelper);
+
+ // Init block part
+ if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
+
+ return 0;
+
+_err:
+ tsdbDestroyHelper(pHelper);
+ return -1;
+}
+
+// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------
+int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
+ return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER);
+}
+
+int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
+ return tsdbInitHelper(pHelper, pRepo, TSDB_WRITE_HELPER);
+}
+
+void tsdbDestroyHelper(SRWHelper *pHelper) {
+ if (pHelper) {
+ tsdbDestroyHelperFile(pHelper);
+ tsdbDestroyHelperTable(pHelper);
+ tsdbDestroyHelperBlock(pHelper);
+ memset((void *)pHelper, 0, sizeof(*pHelper));
+ }
+}
+
+void tsdbResetHelper(SRWHelper *pHelper) {
+ if (pHelper) {
+ // Reset the block part
+ tsdbResetHelperBlockImpl(pHelper);
+
+ // Reset the table part
+ tsdbResetHelperTableImpl(pHelper);
+
+ // Reset the file part
+ tsdbCloseHelperFile(pHelper, false);
+ tsdbResetHelperFileImpl(pHelper);
+
+ pHelper->state = TSDB_HELPER_CLEAR_STATE;
+ }
+}
+
+// ------------ Operations for read/write purpose
+int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
+ ASSERT(pHelper != NULL && pGroup != NULL);
+
+ // Clear the helper object
+ tsdbResetHelper(pHelper);
+
+ ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE);
+
+ // Set the files
+ pHelper->files.fid = pGroup->fileId;
+ pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD];
+ pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA];
+ pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST];
+ if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) {
+ char *fnameDup = strdup(pHelper->files.headF.fname);
+ if (fnameDup == NULL) return -1;
+ char *dataDir = dirname(fnameDup);
+
+ tsdbGetFileName(dataDir, pHelper->files.fid, ".h", pHelper->files.nHeadF.fname);
+ tsdbGetFileName(dataDir, pHelper->files.fid, ".l", pHelper->files.nLastF.fname);
+ free((void *)fnameDup);
+ }
+
+ // Open the files
+ if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err;
+ if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) {
+ if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err;
+ if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err;
+
+ // Create and open .h
+ if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1;
+ size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM);
+ if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, tsize) < tsize) goto _err;
+
+ // Create and open .l file if should
+ if (tsdbShouldCreateNewLast(pHelper)) {
+ if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err;
+ if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) goto _err;
+ }
+ } else {
+ if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err;
+ if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err;
+ }
+
+ helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN);
+
+ return tsdbLoadCompIdx(pHelper, NULL);
+
+ _err:
+ return -1;
+}
+
+int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
+ if (pHelper->files.headF.fd > 0) {
+ close(pHelper->files.headF.fd);
+ pHelper->files.headF.fd = -1;
+ }
+ if (pHelper->files.dataF.fd > 0) {
+ close(pHelper->files.dataF.fd);
+ pHelper->files.dataF.fd = -1;
+ }
+ if (pHelper->files.lastF.fd > 0) {
+ close(pHelper->files.lastF.fd);
+ pHelper->files.lastF.fd = -1;
+ }
+ if (pHelper->files.nHeadF.fd > 0) {
+ close(pHelper->files.nHeadF.fd);
+ pHelper->files.nHeadF.fd = -1;
+ if (hasError) {
+ remove(pHelper->files.nHeadF.fname);
+ } else {
+ rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname);
+ pHelper->files.headF.info = pHelper->files.nHeadF.info;
+ }
+ }
+
+ if (pHelper->files.nLastF.fd > 0) {
+ close(pHelper->files.nLastF.fd);
+ pHelper->files.nLastF.fd = -1;
+ if (hasError) {
+ remove(pHelper->files.nLastF.fname);
+ } else {
+ rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname);
+ pHelper->files.lastF.info = pHelper->files.nLastF.info;
+ }
+ }
+ return 0;
+}
+
+void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
+ ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD));
+
+ // Clear members and state used by previous table
+ tsdbResetHelperTable(pHelper);
+ ASSERT(helperHasState(pHelper, (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)));
+
+ pHelper->tableInfo.tid = pTable->tableId.tid;
+ pHelper->tableInfo.uid = pTable->tableId.uid;
+ pHelper->tableInfo.sversion = pTable->sversion;
+ STSchema *pSchema = tsdbGetTableSchema(pRepo->tsdbMeta, pTable);
+
+ tdInitDataCols(pHelper->pDataCols[0], pSchema);
+ tdInitDataCols(pHelper->pDataCols[1], pSchema);
+
+ SCompIdx *pIdx = pHelper->pCompIdx + pTable->tableId.tid;
+ if (pIdx->offset > 0 && pIdx->hasLast) {
+ pHelper->hasOldLastBlock = true;
+ }
+
+ helperSetState(pHelper, TSDB_HELPER_TABLE_SET);
+ ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1));
+}
+
+/**
+ * Write part of of points from pDataCols to file
+ *
+ * @return: number of points written to file successfully
+ * -1 for failure
+ */
+int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
+ ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
+ ASSERT(pDataCols->numOfPoints > 0);
+
+ SCompBlock compBlock;
+ int rowsToWrite = 0;
+ TSKEY keyFirst = dataColsKeyFirst(pDataCols);
+
+ ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD));
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose
+
+ // Load the SCompInfo part if neccessary
+ ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET));
+ if (tsdbLoadCompInfo(pHelper, NULL) < 0) goto _err;
+
+ if (pIdx->offset == 0 || (!pIdx->hasLast && keyFirst > pIdx->maxKey)) { // Just append as a super block
+ ASSERT(pHelper->hasOldLastBlock == false);
+ rowsToWrite = pDataCols->numOfPoints;
+ SFile *pWFile = NULL;
+ bool isLast = false;
+
+ if (rowsToWrite >= pHelper->config.minRowsPerFileBlock) {
+ pWFile = &(pHelper->files.dataF);
+ } else {
+ isLast = true;
+ pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF);
+ }
+
+ if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast, true) < 0) goto _err;
+
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks) < 0) goto _err;
+ } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block
+ SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks),
+ pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE);
+
+ int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfSuperBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks);
+
+ if (pCompBlock == NULL) { // No key overlap, must has last block, just merge with the last block
+ ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last);
+ rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
+ if (rowsToWrite < 0) goto _err;
+ } else { // Has key overlap
+
+ if (compareKeyBlock((void *)(&keyFirst), (void *)pCompBlock) == 0) {
+ // Key overlap with the block, must merge with the block
+
+ rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
+ if (rowsToWrite < 0) goto _err;
+ } else { // Save as a super block in the middle
+ rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1);
+ ASSERT(rowsToWrite > 0);
+ if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err;
+ if (tsdbInsertSuperBlock(pHelper, pCompBlock, blkIdx) < 0) goto _err;
+ }
+ }
+ }
+
+ return rowsToWrite;
+
+_err:
+ return -1;
+}
+
+int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
+ ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+ SCompBlock compBlock;
+ if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) {
+ if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1;
+
+ SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfSuperBlocks - 1;
+ ASSERT(pCompBlock->last);
+
+ if (pCompBlock->numOfSubBlocks > 1) {
+ if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfSuperBlocks - 1), NULL) < 0) return -1;
+ ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 &&
+ pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock);
+ if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
+ pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0)
+ return -1;
+
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks - 1) < 0) return -1;
+
+ } else {
+ if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) return -1;
+ pCompBlock->offset = lseek(pHelper->files.nLastF.fd, 0, SEEK_END);
+ if (pCompBlock->offset < 0) return -1;
+
+ if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, pCompBlock->len) < pCompBlock->len)
+ return -1;
+ }
+
+ pHelper->hasOldLastBlock = false;
+ }
+
+ return 0;
+}
+
+int tsdbWriteCompInfo(SRWHelper *pHelper) {
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+ if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
+ if (pIdx->offset > 0) {
+ pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+ if (pIdx->offset < 0) return -1;
+ ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
+
+ if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) return -1;
+ }
+ } else {
+ pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER;
+ pHelper->pCompInfo->uid = pHelper->tableInfo.uid;
+ ASSERT((pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0);
+ taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len);
+ pIdx->offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END);
+ if (pIdx->offset < 0) return -1;
+ ASSERT(pIdx->offset >= tsizeof(pHelper->pCompIdx));
+
+ if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1;
+ }
+
+ return 0;
+}
+
+int tsdbWriteCompIdx(SRWHelper *pHelper) {
+ ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER);
+ if (lseek(pHelper->files.nHeadF.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+
+ ASSERT(tsizeof(pHelper->pCompIdx) == sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM));
+ taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx));
+
+ if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx))
+ return -1;
+ return 0;
+}
+
+int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
+ ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN);
+
+ if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) {
+ // If not load from file, just load it in object
+ int fd = pHelper->files.headF.fd;
+
+ if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
+ if (tread(fd, (void *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx)) < tsizeof(pHelper->pCompIdx)) return -1;
+ if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pCompIdx), tsizeof((void *)pHelper->pCompIdx))) {
+ // TODO: File is broken, try to deal with it
+ return -1;
+ }
+ }
+ helperSetState(pHelper, TSDB_HELPER_IDX_LOAD);
+
+ // Copy the memory for outside usage
+ if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx));
+
+ return 0;
+}
+
+int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) {
+ ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET));
+
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+
+ int fd = pHelper->files.headF.fd;
+
+ if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
+ if (pIdx->offset > 0) {
+ if (lseek(fd, pIdx->offset, SEEK_SET) < 0) return -1;
+
+ pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len);
+ if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) return -1;
+ if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) return -1;
+ }
+
+ helperSetState(pHelper, TSDB_HELPER_INFO_LOAD);
+ }
+
+ if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len);
+
+ return 0;
+}
+
+int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
+ ASSERT(pCompBlock->numOfSubBlocks <= 1);
+ int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
+
+ if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) return -1;
+
+ size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
+ pHelper->pCompData = trealloc((void *)pHelper->pCompData, tsize);
+ if (pHelper->pCompData == NULL) return -1;
+ if (tread(fd, (void *)pHelper->pCompData, tsize) < tsize) return -1;
+
+ ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols);
+
+ if (target) memcpy(target, pHelper->pCompData, tsize);
+
+ return 0;
+}
+
+static int comparColIdCompCol(const void *arg1, const void *arg2) {
+ return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId;
+}
+
+static int comparColIdDataCol(const void *arg1, const void *arg2) {
+ return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId;
+}
+
+static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) {
+ size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols;
+ if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1;
+ if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1;
+
+ return 0;
+}
+
+static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds,
+ SDataCols *pDataCols) {
+ if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1;
+ int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
+
+ void *ptr = NULL;
+ for (int i = 0; i < numOfColIds; i++) {
+ int16_t colId = colIds[i];
+
+ ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), comparColIdCompCol);
+ if (ptr == NULL) continue;
+ SCompCol *pCompCol = (SCompCol *)ptr;
+
+ ptr = bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol);
+ ASSERT(ptr != NULL);
+ SDataCol *pDataCol = (SDataCol *)ptr;
+
+ pDataCol->len = pCompCol->len;
+ if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1;
+ }
+
+ return 0;
+}
+
+// Load specific column data from file
+int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) {
+ SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+
+ ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block
+
+ int numOfSubBlocks = pCompBlock->numOfSubBlocks;
+ SCompBlock *pStartBlock =
+ (numOfSubBlocks == 1) ? pCompBlock : (SCompBlock *)((char *)pHelper->pCompInfo->blocks + pCompBlock->offset);
+
+ if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pDataCols) < 0) return -1;
+ for (int i = 1; i < numOfSubBlocks; i++) {
+ pStartBlock++;
+ if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pHelper->pDataCols[1]) < 0) return -1;
+ tdMergeDataCols(pDataCols, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints);
+ }
+
+ return 0;
+}
+
+/**
+ * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1)
+ */
+static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
+ ASSERT(pCompBlock->numOfSubBlocks <= 1);
+
+ SCompData *pCompData = (SCompData *)malloc(pCompBlock->len);
+ if (pCompData == NULL) return -1;
+
+ int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
+ if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
+ if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err;
+ ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
+
+ // TODO : check the checksum
+ size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
+ if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
+ for (int i = 0; i < pCompData->numOfCols; i++) {
+ // TODO: check the data checksum
+ // if (!taosCheckChecksumWhole())
+ }
+
+ ASSERT(pCompBlock->numOfCols == pCompData->numOfCols);
+
+ pDataCols->numOfPoints = pCompBlock->numOfPoints;
+
+ int ccol = 0, dcol = 0;
+ while (true) {
+ if (ccol >= pDataCols->numOfCols) {
+ // TODO: Fill rest NULL
+ break;
+ }
+ if (dcol >= pCompData->numOfCols) break;
+
+ SCompCol *pCompCol = &(pCompData->cols[ccol]);
+ SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+
+ if (pCompCol->colId == pDataCol->colId) {
+ // TODO: uncompress
+ memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tsize + pCompCol->offset), pCompCol->len);
+ ccol++;
+ dcol++;
+ } else if (pCompCol->colId > pDataCol->colId) {
+ // TODO: Fill NULL
+ dcol++;
+ } else {
+ ccol++;
+ }
+ }
+
+ tfree(pCompData);
+ return 0;
+
+_err:
+ tfree(pCompData);
+ return -1;
+}
+
+// Load the whole block data
+int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target) {
+ // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+
+ int numOfSubBlock = pCompBlock->numOfSubBlocks;
+ if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)((char *)pHelper->pCompInfo + pCompBlock->offset);
+
+ tdResetDataCols(pHelper->pDataCols[0]);
+ if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err;
+ for (int i = 1; i < numOfSubBlock; i++) {
+ tdResetDataCols(pHelper->pDataCols[1]);
+ pCompBlock++;
+ if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[1]) < 0) goto _err;
+ if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints) < 0) goto _err;
+ }
+
+ // if (target) TODO
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+// static int tsdbCheckHelperCfg(SHelperCfg *pCfg) {
+// // TODO
+// return 0;
+// }
+
+// static void tsdbClearHelperFile(SHelperFile *pHFile) {
+// pHFile->fid = -1;
+// if (pHFile->headF.fd > 0) {
+// close(pHFile->headF.fd);
+// pHFile->headF.fd = -1;
+// }
+// if (pHFile->dataF.fd > 0) {
+// close(pHFile->dataF.fd);
+// pHFile->dataF.fd = -1;
+// }
+// if (pHFile->lastF.fd > 0) {
+// close(pHFile->lastF.fd);
+// pHFile->lastF.fd = -1;
+// }
+// if (pHFile->nHeadF.fd > 0) {
+// close(pHFile->nHeadF.fd);
+// pHFile->nHeadF.fd = -1;
+// }
+// if (pHFile->nLastF.fd > 0) {
+// close(pHFile->nLastF.fd);
+// pHFile->nLastF.fd = -1;
+// }
+
+// }
+
+static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
+ ASSERT(pHelper->files.lastF.fd > 0);
+ struct stat st;
+ fstat(pHelper->files.lastF.fd, &st);
+ if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true;
+ return false;
+}
+
+static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock,
+ bool isLast, bool isSuperBlock) {
+ ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
+ rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
+
+ SCompData *pCompData = NULL;
+ int64_t offset = 0;
+
+ offset = lseek(pFile->fd, 0, SEEK_END);
+ if (offset < 0) goto _err;
+
+ pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols + sizeof(TSCKSUM));
+ if (pCompData == NULL) goto _err;
+
+ int nColsNotAllNull = 0;
+ int32_t toffset = 0;
+ for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
+ SDataCol *pDataCol = pDataCols->cols + ncol;
+ SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
+
+ if (0) {
+ // TODO: all data to commit are NULL
+ continue;
+ }
+
+ // Compress the data here
+ {
+ // TODO
+ }
+
+ pCompCol->colId = pDataCol->colId;
+ pCompCol->type = pDataCol->type;
+ pCompCol->len = TYPE_BYTES[pCompCol->type] * rowsToWrite; // TODO: change it
+ pCompCol->offset = toffset;
+ nColsNotAllNull++;
+
+ toffset += pCompCol->len;
+ }
+
+ ASSERT(nColsNotAllNull > 0 && nColsNotAllNull <= pDataCols->numOfCols);
+
+ pCompData->delimiter = TSDB_FILE_DELIMITER;
+ pCompData->uid = pHelper->tableInfo.uid;
+ pCompData->numOfCols = nColsNotAllNull;
+
+ // Write SCompData + SCompCol part
+ size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
+ taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
+ if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err;
+ // Write true data part
+ int nCompCol = 0;
+ for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
+ ASSERT(nCompCol < nColsNotAllNull);
+
+ SDataCol *pDataCol = pDataCols->cols + ncol;
+ SCompCol *pCompCol = pCompData->cols + nCompCol;
+
+ if (pDataCol->colId == pCompCol->colId) {
+ if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err;
+ tsize += pCompCol->len;
+ nCompCol++;
+ }
+ }
+
+ pCompBlock->last = isLast;
+ pCompBlock->offset = offset;
+ pCompBlock->algorithm = pHelper->config.compress;
+ pCompBlock->numOfPoints = rowsToWrite;
+ pCompBlock->sversion = pHelper->tableInfo.sversion;
+ pCompBlock->len = (int32_t)tsize;
+ pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
+ pCompBlock->numOfCols = nColsNotAllNull;
+ pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
+ pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
+
+ tfree(pCompData);
+ return 0;
+
+ _err:
+ tfree(pCompData);
+ return -1;
+}
+
+static int compareKeyBlock(const void *arg1, const void *arg2) {
+ TSKEY key = *(TSKEY *)arg1;
+ SCompBlock *pBlock = (SCompBlock *)arg2;
+
+ if (key < pBlock->keyFirst) {
+ return -1;
+ } else if (key > pBlock->keyLast) {
+ return 1;
+ }
+
+ return 0;
+}
+
+// static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) {
+// return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2));
+// }
+
+// Merge the data with a block in file
+static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) {
+ // TODO: set pHelper->hasOldBlock
+ int rowsWritten = 0;
+ SCompBlock compBlock = {0};
+
+ ASSERT(pDataCols->numOfPoints > 0);
+ TSKEY keyFirst = dataColsKeyFirst(pDataCols);
+
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+ ASSERT(blkIdx < pIdx->numOfSuperBlocks);
+
+ // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+ ASSERT(blockAtIdx(pHelper, blkIdx)->numOfSubBlocks >= 1);
+ ASSERT(keyFirst >= blockAtIdx(pHelper, blkIdx)->keyFirst);
+ // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
+
+ if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
+ ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfSuperBlocks-1);
+ int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface
+
+ rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints);
+ if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) &&
+ (blockAtIdx(pHelper, blkIdx)->numOfPoints + rowsWritten < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd) > 0) {
+ if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0)
+ goto _err;
+ if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
+ } else {
+ // Load
+ if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
+ ASSERT(pHelper->pDataCols[0]->numOfPoints == blockAtIdx(pHelper, blkIdx)->numOfPoints);
+ // Merge
+ if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err;
+ // Write
+ SFile *pWFile = NULL;
+ bool isLast = false;
+ if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.minRowsPerFileBlock) {
+ pWFile = &(pHelper->files.dataF);
+ } else {
+ isLast = true;
+ pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF);
+ }
+ if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0],
+ pHelper->pDataCols[0]->numOfPoints, &compBlock, isLast, true) < 0)
+ goto _err;
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err;
+ }
+
+ ASSERT(pHelper->hasOldLastBlock);
+ pHelper->hasOldLastBlock = false;
+ } else {
+ // Key must overlap with the block
+ ASSERT(keyFirst <= blockAtIdx(pHelper, blkIdx)->keyLast);
+
+ TSKEY keyLimit =
+ (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1;
+
+ // rows1: number of rows must merge in this block
+ int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast);
+ // rows2: max nuber of rows the block can have more
+ int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfPoints;
+ // rows3: number of rows between this block and the next block
+ int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit);
+
+ ASSERT(rows3 >= rows1);
+
+ if ((rows2 >= rows1) &&
+ (( blockAtIdx(pHelper, blkIdx)->last) ||
+ ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) {
+ rowsWritten = rows1;
+ bool isLast = false;
+ SFile *pFile = NULL;
+
+ if (blockAtIdx(pHelper, blkIdx)->last) {
+ isLast = true;
+ pFile = &(pHelper->files.lastF);
+ } else {
+ pFile = &(pHelper->files.dataF);
+ }
+
+ if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, rows1, &compBlock, isLast, false) < 0) goto _err;
+ if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
+ } else { // Load-Merge-Write
+ // Load
+ if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
+ if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false;
+
+ rowsWritten = rows3;
+
+ int iter1 = 0; // iter over pHelper->pDataCols[0]
+ int iter2 = 0; // iter over pDataCols
+ int round = 0;
+ // tdResetDataCols(pHelper->pDataCols[1]);
+ while (true) {
+ if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) break;
+ tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pDataCols, &iter2, pHelper->config.maxRowsPerFileBlock * 4 / 5);
+ ASSERT(pHelper->pDataCols[1]->numOfPoints > 0);
+ if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1],
+ pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0)
+ goto _err;
+ if (round == 0) {
+ tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx);
+ } else {
+ tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx);
+ }
+ round++;
+ blkIdx++;
+ // TODO: the blkIdx here is not correct
+
+ // if (iter1 >= pHelper->pDataCols[0]->numOfPoints && iter2 >= rows3) {
+ // if (pHelper->pDataCols[1]->numOfPoints > 0) {
+ // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1],
+ // pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0)
+ // goto _err;
+ // // TODO: the blkIdx here is not correct
+ // tsdbAddSubBlock(pHelper, &compBlock, blkIdx, pHelper->pDataCols[1]->numOfPoints);
+ // }
+ // }
+
+ // TSKEY key1 = iter1 >= pHelper->pDataCols[0]->numOfPoints
+ // ? INT64_MAX
+ // : ((int64_t *)(pHelper->pDataCols[0]->cols[0].pData))[iter1];
+ // TSKEY key2 = iter2 >= rowsWritten ? INT64_MAX : ((int64_t *)(pDataCols->cols[0].pData))[iter2];
+
+ // if (key1 < key2) {
+ // for (int i = 0; i < pDataCols->numOfCols; i++) {
+ // SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
+ // memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints),
+ // ((char *)pHelper->pDataCols[0]->cols[i].pData + TYPE_BYTES[pDataCol->type] * iter1),
+ // TYPE_BYTES[pDataCol->type]);
+ // }
+ // pHelper->pDataCols[1]->numOfPoints++;
+ // iter1++;
+ // } else if (key1 == key2) {
+ // // TODO: think about duplicate key cases
+ // ASSERT(false);
+ // } else {
+ // for (int i = 0; i < pDataCols->numOfCols; i++) {
+ // SDataCol *pDataCol = pHelper->pDataCols[1]->cols + i;
+ // memcpy(((char *)pDataCol->pData + TYPE_BYTES[pDataCol->type] * pHelper->pDataCols[1]->numOfPoints),
+ // ((char *)pDataCols->cols[i].pData +
+ // TYPE_BYTES[pDataCol->type] * iter2),
+ // TYPE_BYTES[pDataCol->type]);
+ // }
+ // pHelper->pDataCols[1]->numOfPoints++;
+ // iter2++;
+ // }
+
+ // if (pHelper->pDataCols[0]->numOfPoints >= pHelper->config.maxRowsPerFileBlock * 4 / 5) {
+ // if (tsdbWriteBlockToFile(pHelper, &pHelper->files.dataF, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfPoints, &compBlock, false, true) < 0) goto _err;
+ // // TODO: blkIdx here is not correct, fix it
+ // tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx);
+
+ // tdResetDataCols(pHelper->pDataCols[1]);
+ // }
+ }
+ }
+ }
+
+ return rowsWritten;
+
+ _err:
+ return -1;
+}
+
+static int compTSKEY(const void *key1, const void *key2) { return ((TSKEY *)key1 - (TSKEY *)key2); }
+
+static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
+
+ if (tsizeof((void *)pHelper->pCompInfo) <= esize) {
+ size_t tsize = esize + sizeof(SCompBlock) * 16;
+ pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize);
+ if (pHelper->pCompInfo == NULL) return -1;
+ }
+
+ return 0;
+}
+
+static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+
+ ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfSuperBlocks);
+ ASSERT(pCompBlock->numOfSubBlocks == 1);
+
+ // Adjust memory if no more room
+ if (pIdx->len == 0) pIdx->len = sizeof(SCompData) + sizeof(TSCKSUM);
+ if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err;
+
+ // Change the offset
+ for (int i = 0; i < pIdx->numOfSuperBlocks; i++) {
+ SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
+ if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock);
+ }
+
+ // Memmove if needed
+ int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx);
+ if (tsize > 0) {
+ ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < tsizeof(pHelper->pCompInfo));
+ ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= tsizeof(pHelper->pCompInfo));
+ memmove((void *)((char *)pHelper->pCompInfo + sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)),
+ (void *)((char *)pHelper->pCompInfo + sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize);
+ }
+ pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock;
+
+ pIdx->numOfSuperBlocks++;
+ pIdx->len += sizeof(SCompBlock);
+ ASSERT(pIdx->len <= tsizeof(pHelper->pCompInfo));
+ pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast;
+ pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last;
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
+ ASSERT(pCompBlock->numOfSubBlocks == 0);
+
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+ ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+
+ SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+ ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS);
+
+ size_t spaceNeeded =
+ (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock);
+ if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err;
+
+ pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+
+ // Add the sub-block
+ if (pSCompBlock->numOfSubBlocks > 1) {
+ size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len);
+ if (tsize > 0) {
+ memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)),
+ (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize);
+
+ for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) {
+ SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
+ if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock);
+ }
+ }
+
+
+ *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock;
+
+ pSCompBlock->numOfSubBlocks++;
+ ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
+ pSCompBlock->len += sizeof(SCompBlock);
+ pSCompBlock->numOfPoints += rowsAdded;
+ pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
+ pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
+ pIdx->len += sizeof(SCompBlock);
+ } else { // Need to create two sub-blocks
+ void *ptr = NULL;
+ for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) {
+ SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
+ if (pTCompBlock->numOfSubBlocks > 1) {
+ ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len);
+ break;
+ }
+ }
+
+ if (ptr == NULL) ptr = (void *)((char *)(pHelper->pCompInfo) + pIdx->len - sizeof(TSCKSUM));
+
+ size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo));
+ if (tsize > 0) {
+ memmove((void *)((char *)ptr + sizeof(SCompBlock) * 2), ptr, tsize);
+ for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) {
+ SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
+ if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2);
+ }
+ }
+
+ ((SCompBlock *)ptr)[0] = *pSCompBlock;
+ ((SCompBlock *)ptr)[0].numOfSubBlocks = 0;
+
+ ((SCompBlock *)ptr)[1] = *pCompBlock;
+
+ pSCompBlock->numOfSubBlocks = 2;
+ pSCompBlock->numOfPoints += rowsAdded;
+ pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
+ pSCompBlock->len = sizeof(SCompBlock) * 2;
+ pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
+ pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
+
+ pIdx->len += (sizeof(SCompBlock) * 2);
+ }
+
+ pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast;
+ pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last;
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
+ ASSERT(pCompBlock->numOfSubBlocks == 1);
+
+ SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
+
+ ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+
+ SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
+
+ ASSERT(pSCompBlock->numOfSubBlocks >= 1);
+
+ // Delete the sub blocks it has
+ if (pSCompBlock->numOfSubBlocks > 1) {
+ size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len);
+ if (tsize > 0) {
+ memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset),
+ (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize);
+ }
+
+ for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) {
+ SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
+ if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks);
+ }
+
+ pIdx->len -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks);
+ }
+
+ *pSCompBlock = *pCompBlock;
+
+ pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast;
+ pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last;
+
+ return 0;
+}
+
+// Get the number of rows in range [minKey, maxKey]
+static int tsdbGetRowsInRange(SDataCols *pDataCols, int minKey, int maxKey) {
+ if (pDataCols->numOfPoints == 0) return 0;
+
+ ASSERT(minKey <= maxKey);
+ TSKEY keyFirst = dataColsKeyFirst(pDataCols);
+ TSKEY keyLast = dataColsKeyLast(pDataCols);
+ ASSERT(keyFirst <= keyLast);
+
+ if (minKey > keyLast || maxKey < keyFirst) return 0;
+
+ void *ptr1 = taosbsearch((void *)&minKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY),
+ compTSKEY, TD_GE);
+ ASSERT(ptr1 != NULL);
+
+ void *ptr2 = taosbsearch((void *)&maxKey, (void *)pDataCols->cols[0].pData, pDataCols->numOfPoints, sizeof(TSKEY),
+ compTSKEY, TD_LE);
+ ASSERT(ptr2 != NULL);
+
+ if ((TSKEY *)ptr2 - (TSKEY *)ptr1 < 0) return 0;
+
+ return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1;
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index c9b3a655c747dfd474055a733e31de519f2e2ce0..ff947231c236a42114df4582c371f2c40ffe854c 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -120,6 +120,7 @@ typedef struct STsdbQueryHandle {
SFileGroup* pFileGroup;
SFileGroupIter fileIter;
SCompIdx* compIndex;
+ SRWHelper rhelper;
} STsdbQueryHandle;
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
@@ -134,7 +135,7 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
pCompBlockLoadInfo->fileListIndex = -1;
}
-tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
+TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
// todo 1. filter not exist table
// todo 2. add the reference count for each table that is involved in query
@@ -142,7 +143,8 @@ tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, S
pQueryHandle->order = pCond->order;
pQueryHandle->window = pCond->twindow;
pQueryHandle->pTsdb = tsdb;
- pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)),
+ pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx));
+ tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
pQueryHandle->cur.fid = -1;
@@ -196,7 +198,7 @@ tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, S
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
- return (tsdb_query_handle_t)pQueryHandle;
+ return (TsdbQueryHandleT)pQueryHandle;
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
@@ -288,14 +290,10 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
SFileGroup* fileGroup = pQueryHandle->pFileGroup;
assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0);
- if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) {
- fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY);
- } else {
- assert(FD_VALID(fileGroup->files[TSDB_FILE_TYPE_HEAD].fd));
- }
+ tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup);
// load all the comp offset value for all tables in this file
- tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables
+ // tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables
*numOfBlocks = 0;
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@@ -303,7 +301,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
- SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid];
+ SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid];
if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file
continue;//no data blocks in the file belongs to pCheckInfo->pTable
} else {
@@ -317,8 +315,13 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
pCheckInfo->compSize = compIndex->len;
}
- tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
-
+ // tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
+ STable* pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->tableId.uid);
+ assert(pTable != NULL);
+
+ tsdbSetHelperTable(&pQueryHandle->rhelper, pTable, pQueryHandle->pTsdb);
+
+ tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
TSKEY s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey);
@@ -409,12 +412,12 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj));
- SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA];
- if (pFile->fd == FD_INITIALIZER) {
- pFile->fd = open(pFile->fname, O_RDONLY);
- }
+ // SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA];
+ // if (pFile->fd == FD_INITIALIZER) {
+ // pFile->fd = open(pFile->fname, O_RDONLY);
+ // }
- if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) {
+ if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) {
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo;
pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup;
@@ -427,9 +430,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
taosArrayDestroy(sa);
tfree(data);
- TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData;
- assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast);
-
return blockLoaded;
}
@@ -596,9 +596,10 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j);
if (pCol->info.colId == colId) {
- SDataCol* pDataCol = &pCols->cols[i];
- memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start,
- pQueryHandle->realNumOfRows * pCol->info.bytes);
+ // SDataCol* pDataCol = &pCols->cols[i];
+// pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start;
+ memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start,
+ pQueryHandle->realNumOfRows * pCol->info.bytes);
break;
}
}
@@ -909,7 +910,7 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
}
// handle data in cache situation
-bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) {
+bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@@ -937,7 +938,6 @@ bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) {
return getDataBlocksInFiles(pQueryHandle);
}
-
}
static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey,
@@ -1009,7 +1009,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max
}
// copy data from cache into data block
-SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) {
+SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
STable* pTable = NULL;
@@ -1067,12 +1067,12 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) {
}
// return null for data block in cache
-int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SDataStatis** pBlockStatis) {
+int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
*pBlockStatis = NULL;
return TSDB_CODE_SUCCESS;
}
-SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) {
+SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
/**
* In the following two cases, the data has been loaded to SColumnInfoData.
* 1. data is from cache, 2. data block is not completed qualified to query time range
@@ -1109,17 +1109,17 @@ SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList
}
}
-int32_t tsdbResetQuery(tsdb_query_handle_t* pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) {
+int32_t tsdbResetQuery(TsdbQueryHandleT* pQueryHandle, STimeWindow* window, TsdbPosT position, int16_t order) {
return 0;
}
-SArray* tsdbRetrieveDataRow(tsdb_query_handle_t* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
+SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
-tsdb_query_handle_t* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
+TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
return NULL;
}
-SArray* tsdbGetTableList(tsdb_query_handle_t* pQueryHandle) { return NULL; }
+SArray* tsdbGetTableList(TsdbQueryHandleT* pQueryHandle) { return NULL; }
static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@@ -1432,7 +1432,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
return TSDB_CODE_SUCCESS;
}
-int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo,
+int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols) {
STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@@ -1474,7 +1474,7 @@ int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond
return ret;
}
-int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
if (pTable == NULL) {
return TSDB_CODE_INVALID_TABLE_ID;
@@ -1491,7 +1491,7 @@ int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pG
return TSDB_CODE_SUCCESS;
}
-void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) {
+void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle;
if (pQueryHandle == NULL) {
return;
@@ -1514,14 +1514,15 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) {
taosArrayDestroy(pQueryHandle->pTableCheckInfo);
tfree(pQueryHandle->compIndex);
- size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
- for (int32_t i = 0; i < cols; ++i) {
- SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
- tfree(pColInfo->pData);
- }
+ size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
+ for (int32_t i = 0; i < cols; ++i) {
+ SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+ tfree(pColInfo->pData);
+ }
taosArrayDestroy(pQueryHandle->pColumns);
tfree(pQueryHandle->pDataBlockInfo);
+ tsdbDestroyHelper(&pQueryHandle->rhelper);
tfree(pQueryHandle);
}
diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp
index eac29af097da67706cbcfff222a6e18c75d2e41c..1fbfa4853b6c60a25fd17582af7e9db3dc74436b 100644
--- a/src/tsdb/tests/tsdbTests.cpp
+++ b/src/tsdb/tests/tsdbTests.cpp
@@ -5,12 +5,84 @@
#include "dataformat.h"
#include "tsdbMain.h"
-double getCurTime() {
+static double getCurTime() {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec + tv.tv_usec * 1E-6;
}
+typedef struct {
+ TsdbRepoT *pRepo;
+ int tid;
+ int64_t uid;
+ int sversion;
+ TSKEY startTime;
+ TSKEY interval;
+ int totalRows;
+ int rowsPerSubmit;
+ STSchema * pSchema;
+} SInsertInfo;
+
+static int insertData(SInsertInfo *pInfo) {
+ SSubmitMsg *pMsg =
+ (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
+ if (pMsg == NULL) return -1;
+ TSKEY start_time = pInfo->startTime;
+
+ // Loop to write data
+ double stime = getCurTime();
+
+ for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) {
+ memset((void *)pMsg, 0, sizeof(SSubmitMsg));
+ SSubmitBlk *pBlock = pMsg->blocks;
+ pBlock->uid = pInfo->uid;
+ pBlock->tid = pInfo->tid;
+ pBlock->sversion = pInfo->sversion;
+ pBlock->len = 0;
+ for (int i = 0; i < pInfo->rowsPerSubmit; i++) {
+ // start_time += 1000;
+ start_time += pInfo->interval;
+ SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
+ tdInitDataRow(row, pInfo->pSchema);
+
+ for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
+ if (j == 0) { // Just for timestamp
+ tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j));
+ } else { // For int
+ int val = 10;
+ tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j));
+ }
+ }
+ pBlock->len += dataRowLen(row);
+ }
+ pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
+ pMsg->numOfBlocks = 1;
+
+ pBlock->len = htonl(pBlock->len);
+ pBlock->numOfRows = htonl(pBlock->numOfRows);
+ pBlock->uid = htobe64(pBlock->uid);
+ pBlock->tid = htonl(pBlock->tid);
+
+ pBlock->sversion = htonl(pBlock->sversion);
+ pBlock->padding = htonl(pBlock->padding);
+
+ pMsg->length = htonl(pMsg->length);
+ pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+ pMsg->compressed = htonl(pMsg->numOfBlocks);
+
+ if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) {
+ tfree(pMsg);
+ return -1;
+ }
+ }
+
+ double etime = getCurTime();
+
+ printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
+ tfree(pMsg);
+ return 0;
+}
+
TEST(TsdbTest, DISABLED_tableEncodeDecode) {
// TEST(TsdbTest, tableEncodeDecode) {
STable *pTable = (STable *)malloc(sizeof(STable));
@@ -48,135 +120,132 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
}
-TEST(TsdbTest, DISABLED_createRepo) {
-// TEST(TsdbTest, createRepo) {
- // STsdbCfg config;
-
- // // 1. Create a tsdb repository
- // tsdbSetDefaultCfg(&config);
- // tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
- // ASSERT_NE(pRepo, nullptr);
-
- // // 2. Create a normal table
- // STableCfg tCfg;
- // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
- // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
-
- // int nCols = 5;
- // STSchema *schema = tdNewSchema(nCols);
-
- // for (int i = 0; i < nCols; i++) {
- // if (i == 0) {
- // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
- // } else {
- // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
- // }
- // }
-
- // tsdbTableSetSchema(&tCfg, schema, true);
-
- // tsdbCreateTable(pRepo, &tCfg);
-
- // // // 3. Loop to write some simple data
- // int nRows = 1;
- // int rowsPerSubmit = 1;
- // int64_t start_time = 1584081000000;
-
- // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit);
-
- // double stime = getCurTime();
-
- // for (int k = 0; k < nRows/rowsPerSubmit; k++) {
- // memset((void *)pMsg, 0, sizeof(SSubmitMsg));
- // SSubmitBlk *pBlock = pMsg->blocks;
- // pBlock->uid = 987607499877672L;
- // pBlock->tid = 0;
- // pBlock->sversion = 0;
- // pBlock->len = 0;
- // for (int i = 0; i < rowsPerSubmit; i++) {
- // // start_time += 1000;
- // start_time += 1000;
- // SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
- // tdInitDataRow(row, schema);
-
- // for (int j = 0; j < schemaNCols(schema); j++) {
- // if (j == 0) { // Just for timestamp
- // tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j));
- // } else { // For int
- // int val = 10;
- // tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
- // }
- // }
- // pBlock->len += dataRowLen(row);
- // }
- // pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
- // pMsg->numOfBlocks = 1;
-
- // pBlock->len = htonl(pBlock->len);
- // pBlock->numOfRows = htonl(pBlock->numOfRows);
- // pBlock->uid = htobe64(pBlock->uid);
- // pBlock->tid = htonl(pBlock->tid);
-
- // pBlock->sversion = htonl(pBlock->sversion);
- // pBlock->padding = htonl(pBlock->padding);
-
- // pMsg->length = htonl(pMsg->length);
- // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
- // pMsg->compressed = htonl(pMsg->numOfBlocks);
-
- // tsdbInsertData(pRepo, pMsg);
- // }
-
- // double etime = getCurTime();
-
- // void *ptr = malloc(150000);
- // free(ptr);
-
- // printf("Spent %f seconds to write %d records\n", etime - stime, nRows);
-
- // tsdbCloseRepo(pRepo);
+// TEST(TsdbTest, DISABLED_createRepo) {
+TEST(TsdbTest, createRepo) {
+ STsdbCfg config;
+ STsdbRepo *repo;
-}
+ // 1. Create a tsdb repository
+ tsdbSetDefaultCfg(&config);
+ ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
-// TEST(TsdbTest, DISABLED_openRepo) {
-TEST(TsdbTest, openRepo) {
- tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
- ASSERT_NE(repo, nullptr);
+ TsdbRepoT *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+ ASSERT_NE(pRepo, nullptr);
- STsdbRepo *pRepo = (STsdbRepo *)repo;
+ // 2. Create a normal table
+ STableCfg tCfg;
+ ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
+ ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
- SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
+ int nCols = 5;
+ STSchema *schema = tdNewSchema(nCols);
- for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
- tsdbOpenFile(&pGroup->files[type], O_RDONLY);
+ for (int i = 0; i < nCols; i++) {
+ if (i == 0) {
+ tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+ } else {
+ tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+ }
}
- SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
- tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
+ tsdbTableSetSchema(&tCfg, schema, true);
- SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
+ tsdbCreateTable(pRepo, &tCfg);
- tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
+ // Insert Some Data
+ SInsertInfo iInfo = {
+ .pRepo = pRepo,
+ .tid = tCfg.tableId.tid,
+ .uid = tCfg.tableId.uid,
+ .sversion = tCfg.sversion,
+ .startTime = 1584081000000,
+ .interval = 1000,
+ .totalRows = 50,
+ .rowsPerSubmit = 1,
+ .pSchema = schema
+ };
- int blockIdx = 0;
- SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
+ ASSERT_EQ(insertData(&iInfo), 0);
- SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
+ // Close the repository
+ tsdbCloseRepo(pRepo);
- tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
+ // Open the repository again
+ pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+ repo = (STsdbRepo *)pRepo;
+ ASSERT_NE(pRepo, nullptr);
- STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
- SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
- tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
+ // Insert more data
+ iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows;
+ iInfo.totalRows = 10;
+ iInfo.pRepo = pRepo;
+ ASSERT_EQ(insertData(&iInfo), 0);
- tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
+ // Close the repository
+ tsdbCloseRepo(pRepo);
- tdResetDataCols(pDataCols);
+ // Open the repository again
+ pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+ repo = (STsdbRepo *)pRepo;
+ ASSERT_NE(pRepo, nullptr);
- tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
+ // Read from file
+ SRWHelper rhelper;
+ tsdbInitReadHelper(&rhelper, repo);
+ SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833);
+ ASSERT_NE(pFGroup, nullptr);
+ ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0);
+
+ STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid);
+ ASSERT_NE(pTable, nullptr);
+ tsdbSetHelperTable(&rhelper, pTable, repo);
+
+ ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0);
+ ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0);
int k = 0;
+}
+
+TEST(TsdbTest, DISABLED_openRepo) {
+// TEST(TsdbTest, openRepo) {
+ // tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
+ // ASSERT_NE(repo, nullptr);
+
+ // STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+ // SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
+
+// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+// tsdbOpenFile(&pGroup->files[type], O_RDONLY);
+// }
+
+// SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
+// tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
+
+// SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
+
+ // tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
+
+// int blockIdx = 0;
+// SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
+
+// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
+
+// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
+
+ // STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
+ // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
+ // tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
+
+// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
+
+ // tdResetDataCols(pDataCols);
+
+ // tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
+
+
+// int k = 0;
}
diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h
index 18a277c7fa19e7819279ee8fd0f0542bedb826e9..ed58c2e60d86b9bb11d71c132295718657b4add4 100644
--- a/src/util/inc/tutil.h
+++ b/src/util/inc/tutil.h
@@ -176,6 +176,13 @@ uint32_t ip2uint(const char *const ip_addr);
void taosSetAllocMode(int mode, const char* path, bool autoDump);
void taosDumpMemoryLeak();
+void * tmalloc(size_t size);
+void * tcalloc(size_t nmemb, size_t size);
+size_t tsizeof(void *ptr);
+void tmemset(void *ptr, int c);
+void * trealloc(void *ptr, size_t size);
+void tzfree(void *ptr);
+
#ifdef TAOS_MEM_CHECK
void * taos_malloc(size_t size, const char *file, uint32_t line);
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 7496ad482bdb49348bc0214f24767f5a12b4aafb..1c55ab945756528f8cf4ff743d654614d6d5edbe 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -618,3 +618,48 @@ char *taosCharsetReplace(char *charsetstr) {
return strdup(charsetstr);
}
+
+void *tmalloc(size_t size) {
+ if (size <= 0) return NULL;
+
+ void *ret = malloc(size + sizeof(size_t));
+ if (ret == NULL) return NULL;
+
+ *(size_t *)ret = size;
+
+ return (void *)((char *)ret + sizeof(size_t));
+}
+
+void *tcalloc(size_t nmemb, size_t size) {
+ size_t tsize = nmemb * size;
+ void * ret = tmalloc(tsize);
+ if (ret == NULL) return NULL;
+
+ tmemset(ret, 0);
+ return ret;
+}
+
+size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
+
+void * trealloc(void *ptr, size_t size) {
+ if (ptr == NULL) return tmalloc(size);
+
+ if (size <= tsizeof(ptr)) return ptr;
+
+ void * tptr = (void *)((char *)ptr - sizeof(size_t));
+ size_t tsize = size + sizeof(size_t);
+ tptr = realloc(tptr, tsize);
+ if (tptr == NULL) return NULL;
+
+ *(size_t *)tptr = size;
+
+ return (void *)((char *)tptr + sizeof(size_t));
+}
+
+void tzfree(void *ptr) {
+ if (ptr) {
+ free((void *)((char *)ptr - sizeof(size_t)));
+ }
+}
\ No newline at end of file
diff --git a/src/util/tests/taosbsearchTest.cpp b/src/util/tests/taosbsearchTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0b250c9ecc277506fc2ba4fd7cac49b2bf57fe7f
--- /dev/null
+++ b/src/util/tests/taosbsearchTest.cpp
@@ -0,0 +1,414 @@
+#include
+
+#include "talgo.h"
+
+static int compareFunc(const void *arg1, const void *arg2) { return (*(int *)arg1) - (*(int *)arg2); }
+
+TEST(testCase, taosbsearch_equal) {
+ // For equal test
+ int key = 3;
+ void *pRet = NULL;
+
+ pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 1 element
+ int array1[1] = {5};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 5;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_NE(pRet, nullptr);
+ ASSERT_EQ(*(int *)pRet, key);
+
+ // 2 element
+ int array2[2] = {3, 6};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(*(int *)pRet, key);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 3 element
+ int array3[3] = {3, 6, 8};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
+ ASSERT_EQ(pRet, nullptr);
+}
+
+TEST(testCase, taosbsearch_greater_or_equal) {
+ // For equal test
+ int key = 3;
+ void *pRet = NULL;
+
+ pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 1 element
+ int array1[1] = {5};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 5);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 5;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
+ ASSERT_NE(pRet, nullptr);
+ ASSERT_EQ(*(int *)pRet, 5);
+
+ // 2 element
+ int array2[2] = {3, 6};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 3 element
+ int array3[3] = {3, 6, 8};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 4 element
+ int array4[4] = {3, 6, 8, 11};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 11;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 13;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 5 element
+ int array5[5] = {3, 6, 8, 11, 15};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 11;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 13;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 15);
+
+ key = 15;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(*(int *)pRet, 15);
+
+ key = 17;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
+ ASSERT_EQ(pRet, nullptr);
+}
+
+TEST(testCase, taosbsearch_less_or_equal) {
+ // For equal test
+ int key = 3;
+ void *pRet = NULL;
+
+ pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ // 1 element
+ int array1[1] = {5};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 5);
+
+ key = 5;
+ pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
+ ASSERT_NE(pRet, nullptr);
+ ASSERT_EQ(*(int *)pRet, 5);
+
+ // 2 element
+ int array2[2] = {3, 6};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ // 3 element
+ int array3[3] = {3, 6, 8};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ // 4 element
+ int array4[4] = {3, 6, 8, 11};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 11;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 13;
+ pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ // 5 element
+ int array5[5] = {3, 6, 8, 11, 15};
+
+ key = 1;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(pRet, nullptr);
+
+ key = 3;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 4;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 3);
+
+ key = 6;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 7;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 6);
+
+ key = 8;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 9;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 8);
+
+ key = 11;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 13;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 11);
+
+ key = 15;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 15);
+
+ key = 17;
+ pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
+ ASSERT_EQ(*(int *)pRet, 15);
+}
\ No newline at end of file
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 0827d90ebcee8732cd2fe867e7baa1084c6cbf5f..472567784f54f0b22265db18a7dffccafa8c8107 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -45,6 +45,9 @@ static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT;
#ifndef _SYNC
tsync_h syncStart(const SSyncInfo *info) { return NULL; }
int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; }
+void syncStop(tsync_h shandle) {}
+int syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
+int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
#endif
static void vnodeInit() {
@@ -88,6 +91,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
STsdbCfg tsdbCfg = {0};
tsdbCfg.precision = pVnodeCfg->cfg.precision;
+ tsdbCfg.compression = -1;
tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions;
tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
@@ -270,10 +274,6 @@ void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
}
-void *vnodeGetTsdb(void *pVnode) {
- return ((SVnodeObj *)pVnode)->tsdb;
-}
-
void vnodeBuildStatusMsg(void *param) {
SDMStatusMsg *pStatus = param;
taosVisitIntHashWithFp(tsDnodeVnodesHash, vnodeBuildVloadMsg, pStatus);
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 7fcd02a102e6a5099b46ff9828c74e4edb2b4d83..94ec707c2ed3cec628776b914f5081f40b8b1928 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -56,8 +56,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
qinfo_t pQInfo = NULL;
if (contLen != 0) {
- void* tsdb = vnodeGetTsdb(pVnode);
- pRet->code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo);
+ pRet->code = qCreateQueryInfo(pVnode->tsdb, pQueryTableMsg, &pQInfo);
SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->qhandle = htobe64((uint64_t) (pQInfo));
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 8e66a427de250ef4139f89845de8e75f09beaf0c..a47624b94fedfd3c131b7822ecba4f7a8cad2f3d 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -147,8 +147,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
- void *pTsdb = vnodeGetTsdb(pVnode);
- code = tsdbCreateTable(pTsdb, &tCfg);
+ code = tsdbCreateTable(pVnode->tsdb, &tCfg);
tfree(pDestSchema);
@@ -211,8 +210,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
- void *pTsdb = vnodeGetTsdb(pVnode);
- code = tsdbAlterTable(pTsdb, &tCfg);
+ code = tsdbAlterTable(pVnode->tsdb, &tCfg);
tfree(pDestSchema);
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
index ab324bcfad10b131238b911e895c99d0a343e5f1..6a9e3e0b9d517885ed8d3b7c449b80bcb303805c 100644
--- a/src/wal/src/walMain.c
+++ b/src/wal/src/walMain.c
@@ -208,6 +208,8 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
}
}
+ closedir(dir);
+
if (count == 0) {
if (pWal->keep) code = walRenew(pWal);
return code;
@@ -248,8 +250,6 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
}
}
- closedir(dir);
-
return code;
}
diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt
index 8189dd4d09d8643b740e4c43a0076286c1c5a9e6..354aa2e05ab2c7ef7de3a145c86dbba2bf5d80f0 100644
--- a/tests/test/c/CMakeLists.txt
+++ b/tests/test/c/CMakeLists.txt
@@ -14,4 +14,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
add_executable(importOneRow importOneRow.c)
target_link_libraries(importOneRow taos_static pthread)
+
+ add_executable(importPerTabe importPerTabe.c)
+ target_link_libraries(importPerTabe taos_static pthread)
ENDIF()
diff --git a/tests/test/c/importPerTabe.c b/tests/test/c/importPerTabe.c
new file mode 100644
index 0000000000000000000000000000000000000000..d7c38c2b6ac1071d5a4d2837abf819d39ac01bd9
--- /dev/null
+++ b/tests/test/c/importPerTabe.c
@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tlog.h"
+#include "ttimer.h"
+#include "tutil.h"
+
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+typedef struct {
+ int64_t rowsPerTable;
+ int64_t pointsPerTable;
+ int64_t tableBeginIndex;
+ int64_t tableEndIndex;
+ int threadIndex;
+ char dbName[32];
+ char stableName[64];
+ pthread_t thread;
+} SInfo;
+
+void *syncTest(void *param);
+void generateRandomPoints();
+void shellParseArgument(int argc, char *argv[]);
+void createDbAndTable();
+void insertData();
+
+int32_t randomData[MAX_RANDOM_POINTS];
+int64_t rowsPerTable = 10000;
+int64_t pointsPerTable = 1;
+int64_t numOfThreads = 1;
+int64_t numOfTablesPerThread = 1;
+char dbName[32] = "db";
+char stableName[64] = "st";
+int32_t cache = 16384;
+int32_t tables = 1000;
+
+int main(int argc, char *argv[]) {
+ shellParseArgument(argc, argv);
+ generateRandomPoints();
+ taos_init();
+ createDbAndTable();
+ insertData();
+}
+
+void createDbAndTable() {
+ dPrint("start to create table");
+
+ TAOS * con;
+ struct timeval systemTime;
+ int64_t st, et;
+ char qstr[64000];
+
+ con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
+ if (con == NULL) {
+ dError("failed to connect to DB, reason:%s", taos_errstr(con));
+ exit(1);
+ }
+
+ sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
+ if (taos_query(con, qstr)) {
+ dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
+ exit(0);
+ }
+
+ sprintf(qstr, "use %s", dbName);
+ if (taos_query(con, qstr)) {
+ dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+ exit(0);
+ }
+
+ gettimeofday(&systemTime, NULL);
+ st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ int64_t totalTables = numOfTablesPerThread * numOfThreads;
+
+ if (strcmp(stableName, "no") != 0) {
+ int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
+ for (int64_t f = 0; f < pointsPerTable; ++f) {
+ len += sprintf(qstr + len, ", f%ld double", f);
+ }
+ sprintf(qstr + len, ") tags(t int)");
+
+ if (taos_query(con, qstr)) {
+ dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+ exit(0);
+ }
+
+ for (int64_t t = 0; t < totalTables; ++t) {
+ sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
+ if (taos_query(con, qstr)) {
+ dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
+ exit(0);
+ }
+ }
+ } else {
+ for (int64_t t = 0; t < totalTables; ++t) {
+ int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
+ for (int64_t f = 0; f < pointsPerTable; ++f) {
+ len += sprintf(qstr + len, ", f%ld double", f);
+ }
+ sprintf(qstr + len, ")");
+
+ if (taos_query(con, qstr)) {
+ dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
+ exit(0);
+ }
+ }
+ }
+
+ gettimeofday(&systemTime, NULL);
+ et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
+}
+
+void insertData() {
+ struct timeval systemTime;
+ int64_t st, et;
+
+ gettimeofday(&systemTime, NULL);
+ st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+
+ dPrint("%d threads are spawned to import data", numOfThreads);
+
+ pthread_attr_t thattr;
+ pthread_attr_init(&thattr);
+ pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+ SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads);
+
+ // Start threads to write
+ for (int i = 0; i < numOfThreads; ++i) {
+ pInfo[i].rowsPerTable = rowsPerTable;
+ pInfo[i].pointsPerTable = pointsPerTable;
+ pInfo[i].tableBeginIndex = i * numOfTablesPerThread;
+ pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread;
+ pInfo[i].threadIndex = i;
+ strcpy(pInfo[i].dbName, dbName);
+ strcpy(pInfo[i].stableName, stableName);
+ pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i));
+ }
+
+ taosMsleep(300);
+ for (int i = 0; i < numOfThreads; i++) {
+ pthread_join(pInfo[i].thread, NULL);
+ }
+
+ gettimeofday(&systemTime, NULL);
+ et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ double seconds = (et - st) / 1000.0 / 1000.0;
+
+ int64_t totalTables = numOfTablesPerThread * numOfThreads;
+ int64_t totalRows = totalTables * rowsPerTable;
+ int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable;
+ double speedOfRows = totalRows / seconds;
+ double speedOfPoints = totalPoints / seconds;
+
+ dPrint(
+ "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf "
+ "PointsPerSecond:%.1lf%s",
+ GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC);
+
+ dPrint("threads exit");
+
+ pthread_attr_destroy(&thattr);
+ free(pInfo);
+}
+
+void *syncTest(void *param) {
+ TAOS * con;
+ SInfo * pInfo = (SInfo *)param;
+ struct timeval systemTime;
+ int64_t st, et;
+ char qstr[65000];
+ int maxBytes = 60000;
+
+ dPrint("thread:%d, start to run", pInfo->threadIndex);
+
+ con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
+ if (con == NULL) {
+ dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
+ exit(1);
+ }
+
+ sprintf(qstr, "use %s", pInfo->dbName);
+ taos_query(con, qstr);
+
+ gettimeofday(&systemTime, NULL);
+ st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+
+ int64_t start = 1587225600000;
+ int64_t interval = 1000; // 1000 ms
+
+ char *sql = qstr;
+ char inserStr[] = "import into";
+ int len = sprintf(sql, "%s", inserStr);
+
+ for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) {
+ len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
+ for (int64_t row = 0; row < pInfo->rowsPerTable; row++) {
+ len += sprintf(sql + len, "(%ld", start - row * interval);
+ for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) {
+ len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]);
+ // len += sprintf(sql + len, ",%ld", row);
+ }
+ len += sprintf(sql + len, ")");
+ if (len > maxBytes) {
+ if (taos_query(con, qstr)) {
+ dError("thread:%d, failed to import table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
+ table, row, taos_errstr(con));
+ }
+
+ // "insert into"
+ len = sprintf(sql, "%s", inserStr);
+
+ // "insert into st1 values"
+ if (row != pInfo->rowsPerTable - 1) {
+ len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
+ }
+ }
+ }
+ }
+
+ if (len != strlen(inserStr)) {
+ taos_query(con, qstr);
+ }
+
+ gettimeofday(&systemTime, NULL);
+ et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+ int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
+ int64_t totalRows = totalTables * pInfo->rowsPerTable;
+ int64_t totalPoints = totalRows * pInfo->pointsPerTable;
+ dPrint("thread:%d, import finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex,
+ (et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints);
+
+ return NULL;
+}
+
+void generateRandomPoints() {
+ for (int r = 0; r < MAX_RANDOM_POINTS; ++r) {
+ randomData[r] = rand() % 1000;
+ }
+}
+
+void printHelp() {
+ char indent[10] = " ";
+ printf("Used to test the performance of TDengine\n After writing all the data in one table, start the next table\n");
+
+ printf("%s%s\n", indent, "-d");
+ printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
+ printf("%s%s\n", indent, "-s");
+ printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table");
+ printf("%s%s\n", indent, "-c");
+ printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
+ printf("%s%s\n", indent, "-r");
+ printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable);
+ printf("%s%s\n", indent, "-p");
+ printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable);
+ printf("%s%s\n", indent, "-t");
+ printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads);
+ printf("%s%s\n", indent, "-n");
+ printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread);
+ printf("%s%s\n", indent, "-tables");
+ printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables);
+ printf("%s%s\n", indent, "-cache");
+ printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache);
+
+ exit(EXIT_SUCCESS);
+}
+
+void shellParseArgument(int argc, char *argv[]) {
+ for (int i = 1; i < argc; i++) {
+ if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+ printHelp();
+ exit(0);
+ } else if (strcmp(argv[i], "-d") == 0) {
+ strcpy(dbName, argv[++i]);
+ } else if (strcmp(argv[i], "-c") == 0) {
+ strcpy(configDir, argv[++i]);
+ } else if (strcmp(argv[i], "-s") == 0) {
+ strcpy(stableName, argv[++i]);
+ } else if (strcmp(argv[i], "-r") == 0) {
+ rowsPerTable = atoi(argv[++i]);
+ } else if (strcmp(argv[i], "-p") == 0) {
+ pointsPerTable = atoi(argv[++i]);
+ } else if (strcmp(argv[i], "-t") == 0) {
+ numOfThreads = atoi(argv[++i]);
+ } else if (strcmp(argv[i], "-n") == 0) {
+ numOfTablesPerThread = atoi(argv[++i]);
+ } else if (strcmp(argv[i], "-tables") == 0) {
+ tables = atoi(argv[++i]);
+ } else if (strcmp(argv[i], "-cache") == 0) {
+ cache = atoi(argv[++i]);
+ } else {
+ }
+ }
+
+ dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC);
+ dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
+ dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
+ dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
+ dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
+ dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
+ dPrint("%sdbName:%s%s", GREEN, dbName, NC);
+ dPrint("%stableName:%s%s", GREEN, stableName, NC);
+ dPrint("%sstart to run%s", GREEN, NC);
+}