提交 f318fdfd 编写于 作者: H Hongze Cheng

more

上级 7d3a57f5
...@@ -41,6 +41,8 @@ typedef struct { ...@@ -41,6 +41,8 @@ typedef struct {
SDataCols * pDataCols; SDataCols * pDataCols;
} SCommitH; } SCommitH;
#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) #define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh))
#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) #define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh)))
#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) #define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet))
...@@ -83,16 +85,16 @@ static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); ...@@ -83,16 +85,16 @@ static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
static int tsdbComparKeyBlock(const void *arg1, const void *arg2); static int tsdbComparKeyBlock(const void *arg1, const void *arg2);
// static int tsdbWriteBlockInfo(SCommitH *pCommih); // static int tsdbWriteBlockInfo(SCommitH *pCommih);
// static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); // static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
// static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
static int tsdbMoveBlock(SCommitH *pCommith, int bidx); static int tsdbMoveBlock(SCommitH *pCommith, int bidx);
static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks);
static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit,
bool isLastOneBlock); bool isLastOneBlock);
static void tsdbResetCommitTable(SCommitH *pCommith); static void tsdbResetCommitTable(SCommitH *pCommith);
static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError);
// static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo);
// static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
// TSKEY maxKey, int maxRows, int8_t update); TSKEY maxKey, int maxRows, int8_t update);
int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
SDiskID did; SDiskID did;
...@@ -1063,19 +1065,18 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { ...@@ -1063,19 +1065,18 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
pBlock = NULL; pBlock = NULL;
} }
} else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) {
// // merge pBlock data and memory data // merge pBlock data and memory data
// if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) {
// TSDB_RUNLOCK_TABLE(pIter->pTable); return -1;
// return -1; }
// }
// bidx++; bidx++;
// if (bidx < nBlocks) { if (bidx < nBlocks) {
// pBlock = pCommith->readh.pBlkInfo->blocks + bidx; pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
// } else { } else {
// pBlock = NULL; pBlock = NULL;
// } }
// nextKey = tsdbNextIterKey(pIter->pIter); nextKey = tsdbNextIterKey(pIter->pIter);
} else { } else {
// // Only commit memory data // // Only commit memory data
// if (pBlock == NULL) { // if (pBlock == NULL) {
...@@ -1329,77 +1330,76 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo ...@@ -1329,77 +1330,76 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo
// return 0; // return 0;
// } // }
// static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
// STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); STsdb * pRepo = TSDB_COMMIT_REPO(pCommith);
// STsdbCfg * pCfg = REPO_CFG(pRepo); STsdbCfg * pCfg = REPO_CFG(pRepo);
// int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks;
// SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
// TSKEY keyLimit; TSKEY keyLimit;
// int16_t colId = 0; int16_t colId = 0;
// SMergeInfo mInfo; SMergeInfo mInfo;
// SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; SBlock subBlocks[TSDB_MAX_SUBBLOCKS];
// SBlock block, supBlock; SBlock block, supBlock;
// SDFile * pDFile; SDFile * pDFile;
// if (bidx == nBlocks - 1) { if (bidx == nBlocks - 1) {
// keyLimit = pCommith->maxKey; keyLimit = pCommith->maxKey;
// } else { } else {
// keyLimit = pBlock[1].keyFirst - 1; keyLimit = pBlock[1].keyFirst - 1;
// } }
// SSkipListIterator titer = *(pIter->pIter); SSkipListIterator titer = *(pIter->pIter);
// if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1;
// tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData,
// pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo);
// if (mInfo.nOperations == 0) { if (mInfo.nOperations == 0) {
// // no new data to insert (all updates denied) // no new data to insert (all updates denied)
// if (tsdbMoveBlock(pCommith, bidx) < 0) { if (tsdbMoveBlock(pCommith, bidx) < 0) {
// return -1; return -1;
// } }
// *(pIter->pIter) = titer; *(pIter->pIter) = titer;
// } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) {
// // Ignore the block // Ignore the block
// ASSERT(0); ASSERT(0);
// *(pIter->pIter) = titer; *(pIter->pIter) = titer;
// } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) {
// // Add a sub-block // Add a sub-block
// tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols,
// pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update,
// pCfg->update, &mInfo); &mInfo);
// if (pBlock->last) { if (pBlock->last) {
// pDFile = TSDB_COMMIT_LAST_FILE(pCommith); pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
// } else { } else {
// pDFile = TSDB_COMMIT_DATA_FILE(pCommith); pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
// } }
// if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1;
// if (pBlock->numOfSubBlocks == 1) { if (pBlock->numOfSubBlocks == 1) {
// subBlocks[0] = *pBlock; subBlocks[0] = *pBlock;
// subBlocks[0].numOfSubBlocks = 0; subBlocks[0].numOfSubBlocks = 0;
// } else { } else {
// memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset),
// sizeof(SBlock) * pBlock->numOfSubBlocks); sizeof(SBlock) * pBlock->numOfSubBlocks);
// } }
// subBlocks[pBlock->numOfSubBlocks] = block; subBlocks[pBlock->numOfSubBlocks] = block;
// supBlock = *pBlock; supBlock = *pBlock;
// supBlock.keyFirst = mInfo.keyFirst; supBlock.keyFirst = mInfo.keyFirst;
// supBlock.keyLast = mInfo.keyLast; supBlock.keyLast = mInfo.keyLast;
// supBlock.numOfSubBlocks++; supBlock.numOfSubBlocks++;
// supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed;
// supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock);
// if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1;
// } else { } else {
// if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1;
// if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1;
// -1; }
// }
// return 0; return 0;
// } }
static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { static int tsdbMoveBlock(SCommitH *pCommith, int bidx) {
SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
...@@ -1454,107 +1454,107 @@ static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const ...@@ -1454,107 +1454,107 @@ static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const
return 0; return 0;
} }
// static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit,
// isLastOneBlock) { bool isLastOneBlock) {
// STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); STsdb * pRepo = TSDB_COMMIT_REPO(pCommith);
// STsdbCfg * pCfg = REPO_CFG(pRepo); STsdbCfg *pCfg = REPO_CFG(pRepo);
// SBlock block; SBlock block;
// SDFile * pDFile; SDFile * pDFile;
// bool isLast; bool isLast;
// int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith);
// int biter = 0; int biter = 0;
// while (true) { while (true) {
// tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows,
// pCfg->update); pCfg->update);
// if (pCommith->pDataCols->numOfRows == 0) break; if (pCommith->pDataCols->numOfRows == 0) break;
// if (isLastOneBlock) { if (isLastOneBlock) {
// if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
// pDFile = TSDB_COMMIT_LAST_FILE(pCommith); pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
// isLast = true; isLast = true;
// } else { } else {
// pDFile = TSDB_COMMIT_DATA_FILE(pCommith); pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
// isLast = false; isLast = false;
// } }
// } else { } else {
// pDFile = TSDB_COMMIT_DATA_FILE(pCommith); pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
// isLast = false; isLast = false;
// } }
// if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1;
// if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1;
// } }
// return 0; return 0;
// } }
// static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
// TSKEY maxKey, int maxRows, int8_t update) { TSKEY maxKey, int maxRows, int8_t update) {
// TSKEY key1 = INT64_MAX; TSKEY key1 = INT64_MAX;
// TSKEY key2 = INT64_MAX; TSKEY key2 = INT64_MAX;
// STSchema *pSchema = NULL; STSchema *pSchema = NULL;
// ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
// tdResetDataCols(pTarget); tdResetDataCols(pTarget);
// while (true) { while (true) {
// key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
// SMemRow row = tsdbNextIterRow(pCommitIter->pIter); SMemRow row = tsdbNextIterRow(pCommitIter->pIter);
// if (row == NULL || memRowKey(row) > maxKey) { if (row == NULL || memRowKey(row) > maxKey) {
// key2 = INT64_MAX; key2 = INT64_MAX;
// } else { } else {
// key2 = memRowKey(row); key2 = memRowKey(row);
// } }
// if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 == INT64_MAX && key2 == INT64_MAX) break;
// if (key1 < key2) { if (key1 < key2) {
// for (int i = 0; i < pDataCols->numOfCols; i++) { for (int i = 0; i < pDataCols->numOfCols; i++) {
// //TODO: dataColAppendVal may fail // TODO: dataColAppendVal may fail
// dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
// pTarget->maxPoints); pTarget->maxPoints);
// } }
// pTarget->numOfRows++; pTarget->numOfRows++;
// (*iter)++; (*iter)++;
// } else if (key1 > key2) { } else if (key1 > key2) {
// if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
// pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
// ASSERT(pSchema != NULL); ASSERT(pSchema != NULL);
// } }
// tdAppendMemRowToDataCol(row, pSchema, pTarget, true); tdAppendMemRowToDataCol(row, pSchema, pTarget, true);
// tSkipListIterNext(pCommitIter->pIter); tSkipListIterNext(pCommitIter->pIter);
// } else { } else {
// if (update != TD_ROW_OVERWRITE_UPDATE) { if (update != TD_ROW_OVERWRITE_UPDATE) {
// //copy disk data // copy disk data
// for (int i = 0; i < pDataCols->numOfCols; i++) { for (int i = 0; i < pDataCols->numOfCols; i++) {
// //TODO: dataColAppendVal may fail // TODO: dataColAppendVal may fail
// dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
// pTarget->maxPoints); pTarget->maxPoints);
// } }
// if(update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++;
// } }
// if (update != TD_ROW_DISCARD_UPDATE) { if (update != TD_ROW_DISCARD_UPDATE) {
// //copy mem data // copy mem data
// if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
// pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
// ASSERT(pSchema != NULL); ASSERT(pSchema != NULL);
// } }
// tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
// } }
// (*iter)++; (*iter)++;
// tSkipListIterNext(pCommitIter->pIter); tSkipListIterNext(pCommitIter->pIter);
// } }
// if (pTarget->numOfRows >= maxRows) break; if (pTarget->numOfRows >= maxRows) break;
// } }
// } }
static void tsdbResetCommitTable(SCommitH *pCommith) { static void tsdbResetCommitTable(SCommitH *pCommith) {
taosArrayClear(pCommith->aSubBlk); taosArrayClear(pCommith->aSubBlk);
...@@ -1573,23 +1573,23 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { ...@@ -1573,23 +1573,23 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) {
tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
} }
// static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) {
// STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); STsdb * pRepo = TSDB_COMMIT_REPO(pCommith);
// STsdbCfg * pCfg = REPO_CFG(pRepo); STsdbCfg *pCfg = REPO_CFG(pRepo);
// int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed;
// ASSERT(mergeRows > 0); ASSERT(mergeRows > 0);
// if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) {
// if (pBlock->last) { if (pBlock->last) {
// if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true;
// } else { } else {
// if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true;
// } }
// } }
// return false; return false;
// } }
// int tsdbApplyRtn(STsdbRepo *pRepo) { // int tsdbApplyRtn(STsdbRepo *pRepo) {
// SRtn rtn; // SRtn rtn;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册