Skip to content

TDengine
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
TDengine
TDengine
提交 dbd8c300 编写于 作者: H Haojun Liao
浏览文件
操作

refactor: do tsdbread refactor.

上级 ce15ee0e
显示空白变更内容
内联 并排
Showing with 989 addition and 783 deletion
include/common/tcommon.h
浏览文件 @ dbd8c300
...@@ -101,17 +101,17 @@ typedef struct SColumnInfoData { ...@@ -101,17 +101,17 @@ typedef struct SColumnInfoData {
} SColumnInfoData; } SColumnInfoData;
typedef struct SQueryTableDataCond { typedef struct SQueryTableDataCond {
// STimeWindow twindow;
uint64_t suid; uint64_t suid;
int32_t order; // desc|asc order to iterate the data block int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols; int32_t numOfCols;
SColumnInfo* colList; SColumnInfo* colList;
bool loadExternalRows; // load external rows or not
int32_t type; // data block load type: int32_t type; // data block load type:
int32_t numOfTWindows; int32_t numOfTWindows;
STimeWindow* twindows; STimeWindow* twindows;
int64_t startVersion; int32_t numOfTables; // number of tables
int64_t endVersion; uint64_t* uidList; // table uid list
int64_t startVersion; // start version
int64_t endVersion; // end version
} SQueryTableDataCond; } SQueryTableDataCond;
void* blockDataDestroy(SSDataBlock* pBlock); void* blockDataDestroy(SSDataBlock* pBlock);
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ dbd8c300
...@@ -116,12 +116,11 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); ...@@ -116,12 +116,11 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur);
// typedef struct STsdb STsdb; // typedef struct STsdb STsdb;
typedef struct STsdbReader STsdbReader; typedef struct STsdbReader STsdbReader;
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 #define BLOCK_LOAD_OFFSET_ORDER 1
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLESEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3 #define BLOCK_LOAD_EXTERN_ORDER 3
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableInfoGroup, uint64_t qId, int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* pTableList, uint64_t qId, uint64_t taskId, STsdbReader** ppReader);
uint64_t taskId, STsdbReader **ppReader);
void tsdbReaderClose(STsdbReader *pReader); void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader);
void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo);
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ dbd8c300
...@@ -14,7 +14,6 @@ ...@@ -14,7 +14,6 @@
*/ */
#include "tsdb.h" #include "tsdb.h"
#define EXTRA_BYTES 2
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
...@@ -24,11 +23,6 @@ ...@@ -24,11 +23,6 @@
.rows = (_block)->numOfRows, \ .rows = (_block)->numOfRows, \
.uid = (_checkInfo)->tableId}) .uid = (_checkInfo)->tableId})
enum {
TSDB_QUERY_TYPE_ALL = 1,
TSDB_QUERY_TYPE_LAST = 2,
};
enum { enum {
TSDB_CACHED_TYPE_NONE = 0, TSDB_CACHED_TYPE_NONE = 0,
TSDB_CACHED_TYPE_LASTROW = 1, TSDB_CACHED_TYPE_LASTROW = 1,
...@@ -41,7 +35,7 @@ typedef struct SQueryFilePos { ...@@ -41,7 +35,7 @@ typedef struct SQueryFilePos {
int32_t pos; int32_t pos;
int64_t lastKey; int64_t lastKey;
int32_t rows; int32_t rows;
bool mixBlock; bool composedBlock;
bool blockCompleted; bool blockCompleted;
STimeWindow win; STimeWindow win;
} SQueryFilePos; } SQueryFilePos;
...@@ -53,39 +47,37 @@ typedef struct SDataBlockLoadInfo { ...@@ -53,39 +47,37 @@ typedef struct SDataBlockLoadInfo {
SArray* pLoadedCols; SArray* pLoadedCols;
} SDataBlockLoadInfo; } SDataBlockLoadInfo;
typedef struct SLoadCompBlockInfo {
int32_t tid; /* table tid */
int32_t fileId;
} SLoadCompBlockInfo;
enum { enum {
CHECKINFO_CHOSEN_MEM = 0, CHECKINFO_CHOSEN_MEM = 0,
CHECKINFO_CHOSEN_IMEM = 1, CHECKINFO_CHOSEN_IMEM = 1,
CHECKINFO_CHOSEN_BOTH = 2 // for update=2(merge case) CHECKINFO_CHOSEN_BOTH = 2 // for update=2(merge case)
}; };
typedef struct STableCheckInfo { typedef struct STableBlockScanInfo {
uint64_t suid; uint64_t uid;
uint64_t tableId;
TSKEY lastKey; TSKEY lastKey;
SBlockIdx blockIdx;
SArray* pBlockList; // block data index list
// SBlockInfo* pCompInfo; // SBlockInfo* pCompInfo;
int32_t compSize; // int32_t compSize;
int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks // int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks
uint8_t chosen : 2; // indicate which iterator should move forward uint8_t chosen : 2; // indicate which iterator should move forward
bool initBuf : 1; // whether to initialize the in-memory skip list iterator or not bool iterInit; // whether to initialize the in-memory skip list iterator or not
STbDataIter* iter; // mem buffer skip list iterator STbDataIter iter; // mem buffer skip list iterator
STbDataIter* iiter; // imem buffer skip list iterator STbDataIter iiter; // imem buffer skip list iterator
} STableCheckInfo; bool memHasVal;
bool imemHasVal;
typedef struct STableBlockInfo { } STableBlockScanInfo;
SBlock* compBlock;
STableCheckInfo* pTableCheckInfo; typedef struct SBlockOrderWrapper {
} STableBlockInfo; int64_t uid;
SBlock* pBlock;
} SBlockOrderWrapper;
typedef struct SBlockOrderSupporter { typedef struct SBlockOrderSupporter {
int32_t numOfTables; int32_t numOfTables;
STableBlockInfo** pDataBlockInfo; SBlockOrderWrapper** pDataBlockInfo;
int32_t* blockIndexArray; int32_t* indexPerTable;
int32_t* numOfBlocksPerTable; int32_t* numOfBlocksPerTable;
} SBlockOrderSupporter; } SBlockOrderSupporter;
...@@ -100,47 +92,92 @@ typedef struct SIOCostSummary { ...@@ -100,47 +92,92 @@ typedef struct SIOCostSummary {
typedef struct SBlockLoadSuppInfo { typedef struct SBlockLoadSuppInfo {
SColumnDataAgg* pstatis; SColumnDataAgg* pstatis;
SColumnDataAgg** plist; SColumnDataAgg** plist;
SArray* defaultLoadColumn; // default load column int16_t* colIdList; // default load column
int32_t* slotIds; // colId to slotId int32_t* slotIds; // colId to slotId
} SBlockLoadSuppInfo; } SBlockLoadSuppInfo;
typedef struct SFileSetIter {
int32_t numOfFiles; // number of total files
int32_t index; // current accessed index in the list
SArray* pFileList; // data file list
} SFileSetIter;
typedef struct SFileDataBlockInfo {
int32_t tbBlockIdx; // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it
int64_t uid;
} SFileDataBlockInfo;
typedef struct SDataBlockIter {
int32_t numOfBlocks;
int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo>
} SDataBlockIter;
typedef struct SFileBlockDumpInfo {
int32_t totalRows;
int32_t rowIndex;
int64_t lastKey;
} SFileBlockDumpInfo;
typedef struct SComposedDataBlock {
bool composed;
int32_t rows;
} SComposedDataBlock;
typedef struct SReaderStatus {
SQueryFilePos cur; // current position
int32_t tableListIndex;
bool loadFromFile; // check file stage
bool initStartPos;
SHashObj* pTableMap; // SHash<STableBlockScanInfo>
int32_t realNumOfRows;
SFileBlockDumpInfo fBlockDumpInfo;
SBlockData fileBlockData;
SFileSetIter fileIter;
SDataBlockIter blockIter;
} SReaderStatus;
struct STsdbReader { struct STsdbReader {
STsdb* pTsdb; STsdb* pTsdb;
uint64_t suid; uint64_t suid;
int16_t order; int16_t order;
SQueryFilePos cur; // current position
STimeWindow window; // the primary query time window that applies to all queries STimeWindow window; // the primary query time window that applies to all queries
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time SSDataBlock* pResBlock;
// SColumnDataAgg** pstatis;// the ptr array list to return to caller int32_t capacity;
int32_t numOfBlocks; SReaderStatus status;
SArray* pColumns; // SArray<SColumnInfoData>
bool locateStart;
int32_t outputCapacity;
int32_t realNumOfRows;
SArray* pTableCheckInfo; // SArray<STableCheckInfo>
int32_t activeIndex;
bool checkFiles; // check file stage
int8_t cachelastrow; // check if last row cached
bool loadExternalRow; // load time window external data rows
bool currentLoadExternalRows; // current load external rows
int32_t loadType; // block load type
char* idStr; // query info handle, for debug purpose char* idStr; // query info handle, for debug purpose
int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows
SDFileSet* pFileGroup;
// SFSIter fileIter;
// SReadH rhelper;
STableBlockInfo* pDataBlockInfo;
SDataCols* pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */
SBlockLoadSuppInfo suppInfo; SBlockLoadSuppInfo suppInfo;
SArray* prev; // previous row which is before than time window SArray* prev; // previous row which is before than time window
SArray* next; // next row which is after the query time window SArray* next; // next row which is after the query time window
SIOCostSummary cost; SIOCostSummary cost;
STSchema* pSchema; STSchema* pSchema;
SDataFReader* pFileReader;
int64_t startVersion;
int64_t endVersion;
#if 0
SFileBlockInfo* pDataBlockInfo;
SDataCols* pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */
// SDFileSet* pFileGroup;
// SFSIter fileIter;
// SReadH rhelper;
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time
// SColumnDataAgg** pstatis;// the ptr array list to return to caller
#endif
}; };
static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
static int tsdbReadRowsFromCache(STableBlockScanInfo* pScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader);
static TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader);
static int32_t doLoadRowsOfIdenticalTsInFileBlock(SBlockData* pData, SFileBlockDumpInfo* pDumpInfo, int64_t ts, SRowMerger *pMerger,
STsdbReader* pReader, STSRow** pRow);
// static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { // static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
// pBlockLoadInfo->slot = -1; // pBlockLoadInfo->slot = -1;
// pBlockLoadInfo->uid = 0; // pBlockLoadInfo->uid = 0;
...@@ -152,70 +189,50 @@ struct STsdbReader { ...@@ -152,70 +189,50 @@ struct STsdbReader {
// pCompBlockLoadInfo->fileId = -1; // pCompBlockLoadInfo->fileId = -1;
// } // }
// static SArray* getColumnIdList(STsdbReader* pTsdbReadHandle) { static int32_t setColumnIdList(STsdbReader* pReader, SSDataBlock* pBlock) {
// size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// assert(numOfCols <= TSDB_MAX_COLUMNS); pReader->suppInfo.colIdList = taosMemoryCalloc(numOfCols, sizeof(int16_t));
if (pReader->suppInfo.colIdList == NULL) {
// SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); return TSDB_CODE_OUT_OF_MEMORY;
// for (int32_t i = 0; i < numOfCols; ++i) { }
// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
// taosArrayPush(pIdList, &pCol->info.colId);
// }
// return pIdList;
// }
// static SArray* getDefaultLoadColumns(STsdbReader* pTsdbReadHandle, bool loadTS) {
// SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle);
// // check if the primary time stamp column needs to load
// int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0);
// // the primary timestamp column does not be included in the the specified load column list, add it
// if (loadTS && colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
// int16_t columnId = PRIMARYKEY_TIMESTAMP_COL_ID;
// taosArrayInsert(pLocalIdList, 0, &columnId);
// }
// return pLocalIdList; for (int32_t i = 0; i < numOfCols; ++i) {
// } SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
pReader->suppInfo.colIdList[i] = pCol->info.colId;
}
static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STableListInfo* pTableList) { return TSDB_CODE_SUCCESS;
// size_t tableSize = taosArrayGetSize(pTableList->pTableList); }
// assert(tableSize >= 1);
// // allocate buffer in order to load data blocks from file static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const uint64_t* idList, int32_t numOfTables) {
// SArray* pTableCheckInfo = taosArrayInit(tableSize, sizeof(STableCheckInfo)); ASSERT(numOfTables >= 1);
// if (pTableCheckInfo == NULL) {
// return NULL;
// }
// // todo apply the lastkey of table check to avoid to load header file // allocate buffer in order to load data blocks from file
// for (int32_t j = 0; j < tableSize; ++j) { // todo use simple hash instead
// STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j); SHashObj* pTableMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pTableMap == NULL) {
return NULL;
}
// STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid}; // todo apply the lastkey of table check to avoid to load header file
// info.suid = pTsdbReadHandle->suid; for (int32_t j = 0; j < numOfTables; ++j) {
// if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j]};
// if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
// info.lastKey = pTsdbReadHandle->window.skey; if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
// } info.lastKey = pTsdbReader->window.skey;
}
// assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey);
// } else { } else {
// info.lastKey = pTsdbReadHandle->window.skey; info.lastKey = pTsdbReader->window.skey;
// } }
// taosArrayPush(pTableCheckInfo, &info); taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
// tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReadHandle, info.tableId, tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, info.uid, info.lastKey,
// info.lastKey, pTsdbReader->idStr);
// pTsdbReadHandle->idStr); }
// }
// // TODO group table according to the tag value. return pTableMap;
// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
// return pTableCheckInfo;
return NULL;
} }
// static void resetCheckInfo(STsdbReader* pTsdbReadHandle) { // static void resetCheckInfo(STsdbReader* pTsdbReadHandle) {
...@@ -224,7 +241,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl ...@@ -224,7 +241,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl
// // todo apply the lastkey of table check to avoid to load header file // // todo apply the lastkey of table check to avoid to load header file
// for (int32_t i = 0; i < numOfTables; ++i) { // for (int32_t i = 0; i < numOfTables; ++i) {
// STableCheckInfo* pCheckInfo = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // STableBlockScanInfo* pCheckInfo = (STableBlockScanInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
// pCheckInfo->lastKey = pTsdbReadHandle->window.skey; // pCheckInfo->lastKey = pTsdbReadHandle->window.skey;
// pCheckInfo->iter = tsdbTbDataIterDestroy(pCheckInfo->iter); // pCheckInfo->iter = tsdbTbDataIterDestroy(pCheckInfo->iter);
// pCheckInfo->iiter = tsdbTbDataIterDestroy(pCheckInfo->iiter); // pCheckInfo->iiter = tsdbTbDataIterDestroy(pCheckInfo->iiter);
...@@ -239,24 +256,24 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl ...@@ -239,24 +256,24 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl
// } // }
// // only one table, not need to sort again // // only one table, not need to sort again
// static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY skey, SArray** psTable) { // static SArray* createCheckInfoFromCheckInfo(STableBlockScanInfo* pCheckInfo, TSKEY skey, SArray** psTable) {
// SArray* pNew = taosArrayInit(1, sizeof(STableCheckInfo)); // SArray* pNew = taosArrayInit(1, sizeof(STableBlockScanInfo));
// STableCheckInfo info = {.lastKey = skey}; // STableBlockScanInfo info = {.lastKey = skey};
// info.tableId = pCheckInfo->tableId; // info.tableId = pCheckInfo->tableId;
// taosArrayPush(pNew, &info); // taosArrayPush(pNew, &info);
// return pNew; // return pNew;
// } // }
// static bool emptyQueryTimewindow(STsdbReader* pTsdbReadHandle) { static bool isEmptyQueryTimeWindow(STsdbReader* pTsdbReader) {
// assert(pTsdbReadHandle != NULL); ASSERT(pTsdbReader != NULL);
// STimeWindow* w = &pTsdbReadHandle->window; STimeWindow* w = &pTsdbReader->window;
// bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); bool asc = ASCENDING_TRAVERSE(pTsdbReader->order);
// return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey));
// } }
// // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // // Update the query time window according to the data time to live(TTL) information, in order to avoid to return
// // the expired data to client, even it is queried already. // // the expired data to client, even it is queried already.
...@@ -267,6 +284,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl ...@@ -267,6 +284,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReader* pTsdbReadHandle, STabl
// return now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick // return now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick
// } // }
// todo remove this
static void setQueryTimewindow(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) { static void setQueryTimewindow(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) {
// pReader->window = pCond->twindows[tWinIdx]; // pReader->window = pCond->twindows[tWinIdx];
...@@ -293,113 +311,140 @@ static void setQueryTimewindow(STsdbReader* pReader, SQueryTableDataCond* pCond, ...@@ -293,113 +311,140 @@ static void setQueryTimewindow(STsdbReader* pReader, SQueryTableDataCond* pCond,
// } // }
} }
static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId, static void checkResultSize(const SQueryTableDataCond* pCond, STsdbReader* pReader) {
STsdbReader** ppReader) { int32_t rowLen = 0;
int32_t code = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) {
STsdbReader* pReader = NULL; rowLen += pCond->colList[i].bytes;
}
// alloc // make sure the output SSDataBlock size be less than 2MB.
pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); int32_t TWOMB = 2 * 1024 * 1024;
if (pReader == NULL) { if (pReader->capacity * rowLen > TWOMB) {
code = TSDB_CODE_OUT_OF_MEMORY; pReader->capacity = TWOMB / rowLen;
}
}
// init file iterator
static int32_t initFileIterator(SFileSetIter* pIter, const STsdbFSState* pFState) {
pIter->index = -1;
pIter->numOfFiles = taosArrayGetSize(pFState->aDFileSet);
pIter->pFileList = taosArrayDup(pFState->aDFileSet);
return TSDB_CODE_SUCCESS;
}
static void resetDataBlockIterator(SDataBlockIter* pIter) {
pIter->numOfBlocks = -1;
}
static bool nextFilesetIterator(SFileSetIter* pIter, int32_t order, STsdbReader* pReader) {
if (pIter->index >= pIter->numOfFiles) {
return false;
}
pIter->index += 1;
// check file the time range of coverage
STimeWindow win = {0};
SDFileSet *pDFile = (SDFileSet *)taosArrayGet(pIter->pFileList, pIter->index);
int32_t code = tsdbDataFReaderOpen(&pReader->pFileReader, pReader->pTsdb, pDFile);
if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _err;
} }
pReader->pTsdb = pVnode->pTsdb; // TODO: pass in pTsdb directly
pReader->suid = pCond->suid;
pReader->order = pCond->order;
pReader->loadType = pCond->type;
pReader->loadExternalRow = pCond->loadExternalRows;
pReader->currentLoadExternalRows = pCond->loadExternalRows;
pReader->type = TSDB_QUERY_TYPE_ALL;
pReader->cur.fid = INT32_MIN;
pReader->cur.win = TSWINDOW_INITIALIZER;
pReader->checkFiles = true;
pReader->activeIndex = 0; // current active table index
pReader->allocSize = 0;
pReader->locateStart = false;
pReader->outputCapacity = 4096; //((STsdb*)tsdb)->config.maxRowsPerFileBlock;
// char buf[128] = {0};
// snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId);
// pReadHandle->idStr = strdup(buf);
// // if (tsdbInitReadH(&pReadHandle->rhelper, pReadHandle->pTsdb) != 0) {
// // goto _end;
// // }
setQueryTimewindow(pReader, pCond, 0); // tsdbGetFidKeyRange(pCfg->days, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
if (pCond->numOfCols > 0) { // current file are not overlapped with query time window, ignore remain files
// int32_t rowLen = 0; if ((ASCENDING_TRAVERSE(order) && win.skey > pReader->window.ekey) ||
// for (int32_t i = 0; i < pCond->numOfCols; ++i) { (!ASCENDING_TRAVERSE(order) && win.ekey < pReader->window.ekey)) {
// rowLen += pCond->colList[i].bytes; tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader,
// } pReader->window.skey, pReader->window.ekey, pReader->idStr);
return false;
}
// // make sure the output SSDataBlock size be less than 2MB. _err:
// int32_t TWOMB = 2 * 1024 * 1024; return false;
// if (pReadHandle->outputCapacity * rowLen > TWOMB) { }
// pReadHandle->outputCapacity = TWOMB / rowLen;
// }
// // allocate buffer in order to load data blocks from file static void initReaderStatus(SReaderStatus* pStatus) {
// pReadHandle->suppInfo.pstatis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg)); pStatus->cur.fid = INT32_MIN;
// if (pReadHandle->suppInfo.pstatis == NULL) { pStatus->cur.win = TSWINDOW_INITIALIZER;
// goto _end; pStatus->initStartPos = false;
// } pStatus->tableListIndex = 0; // current active table index
pStatus->loadFromFile = true;
}
// // todo: use list instead of array? static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, const char* idstr) {
// pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); int32_t code = 0;
// if (pReadHandle->pColumns == NULL) { STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
// goto _end; if (pReader == NULL) {
// } code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
// for (int32_t i = 0; i < pCond->numOfCols; ++i) { initReaderStatus(&pReader->status);
// SColumnInfoData colInfo = {{0}, 0}; pReader->pTsdb = pVnode->pTsdb;
// colInfo.info = pCond->colList[i]; pReader->suid = pCond->suid;
pReader->order = pCond->order;
pReader->capacity = 4096;
pReader->idStr = strdup(idstr);
pReader->startVersion= pCond->startVersion;
pReader->endVersion = pCond->endVersion;
// int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); // todo remove this
// if (code != TSDB_CODE_SUCCESS) { setQueryTimewindow(pReader, pCond, 0);
// goto _end;
// }
// taosArrayPush(pReadHandle->pColumns, &colInfo); if (pCond->numOfCols > 0) {
// } checkResultSize(pCond, pReader);
// pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); // allocate buffer in order to load data blocks from file
pReader->suppInfo.pstatis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg));
if (pReader->suppInfo.pstatis == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
// size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); // todo use new api refactor this
// pReadHandle->suppInfo.slotIds = taosMemoryCalloc(size, sizeof(int32_t)); pReader->pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
// pReadHandle->suppInfo.plist = taosMemoryCalloc(size, POINTER_BYTES); if (pReader->pResBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
} }
// pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); pReader->pResBlock->pDataBlock = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData));
// if (pReadHandle->pDataCols == NULL) { for (int32_t i = 0; i < pCond->numOfCols; ++i) {
// tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); SColumnInfoData colInfo = {{0}, 0};
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; colInfo.info = pCond->colList[i];
// goto _end; taosArrayPush(pReader->pResBlock->pDataBlock, &colInfo);
// } }
// tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); setColumnIdList(pReader, pReader->pResBlock);
// tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); pReader->suppInfo.slotIds = taosMemoryCalloc(pCond->numOfCols, sizeof(int32_t));
pReader->suppInfo.plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES);
}
// return (STsdbReader*)pReadHandle; // todo refactor
STsdbFSState* pFState = pReader->pTsdb->fs->cState;
initFileIterator(&pReader->status.fileIter, pFState);
resetDataBlockIterator(&pReader->status.blockIter);
// _end: // no data in files, let's try buffer in memory
// tsdbReaderClose(pReadHandle); if (pReader->status.fileIter.numOfFiles == 0) {
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; pReader->status.loadFromFile = false;
// return NULL; }
*ppReader = pReader; *ppReader = pReader;
return code; return code;
_err: _end:
// tsdbError(""); tsdbReaderClose(pReader);
*ppReader = NULL; *ppReader = NULL;
return code; return code;
} }
// static int32_t setCurrentSchema(SVnode* pVnode, STsdbReader* pTsdbReadHandle) { // static int32_t setCurrentSchema(SVnode* pVnode, STsdbReader* pTsdbReadHandle) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
// int32_t sversion = 1; // int32_t sversion = 1;
...@@ -463,7 +508,7 @@ _err: ...@@ -463,7 +508,7 @@ _err:
// // pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); // // pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo);
// pTsdbReadHandle->pTableCheckInfo = NULL; // createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, // pTsdbReadHandle->pTableCheckInfo = NULL; // createDataBlockScanInfo(pTsdbReadHandle, groupList, pMeta,
// // &pTable); // // &pTable);
// if (pTsdbReadHandle->pTableCheckInfo == NULL) { // if (pTsdbReadHandle->pTableCheckInfo == NULL) {
// // tsdbReaderClose(pTsdbReadHandle); // // tsdbReaderClose(pTsdbReadHandle);
...@@ -484,7 +529,7 @@ _err: ...@@ -484,7 +529,7 @@ _err:
// return res; // return res;
// } // }
// static bool initTableMemIterator(STsdbReader* pHandle, STableCheckInfo* pCheckInfo) { // static bool initTableMemIterator(STsdbReader* pHandle, STableBlockScanInfo* pCheckInfo) {
// if (pCheckInfo->initBuf) { // if (pCheckInfo->initBuf) {
// return true; // return true;
// } // }
...@@ -566,12 +611,12 @@ _err: ...@@ -566,12 +611,12 @@ _err:
// return true; // return true;
// } // }
// static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { // static void destroyTableMemIterator(STableBlockScanInfo* pCheckInfo) {
// tsdbTbDataIterDestroy(pCheckInfo->iter); // tsdbTbDataIterDestroy(pCheckInfo->iter);
// tsdbTbDataIterDestroy(pCheckInfo->iiter); // tsdbTbDataIterDestroy(pCheckInfo->iiter);
// } // }
// static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) { // static TSKEY extractFirstTraverseKey(STableBlockScanInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) {
// TSDBROW row = {0}; // TSDBROW row = {0};
// STSRow *rmem = NULL, *rimem = NULL; // STSRow *rmem = NULL, *rimem = NULL;
...@@ -621,7 +666,7 @@ _err: ...@@ -621,7 +666,7 @@ _err:
// } // }
// } // }
// static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, // static STSRow* getSRowInTableMem(STableBlockScanInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow,
// TDRowVerT maxVer) { // TDRowVerT maxVer) {
// TSDBROW row; // TSDBROW row;
// STSRow *rmem = NULL, *rimem = NULL; // STSRow *rmem = NULL, *rimem = NULL;
...@@ -685,7 +730,7 @@ _err: ...@@ -685,7 +730,7 @@ _err:
// } // }
// } // }
// static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { // static bool moveToNextRowInMem(STableBlockScanInfo* pCheckInfo) {
// bool hasNext = false; // bool hasNext = false;
// if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) { // if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) {
// if (pCheckInfo->iter != NULL) { // if (pCheckInfo->iter != NULL) {
...@@ -729,7 +774,7 @@ _err: ...@@ -729,7 +774,7 @@ _err:
// assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); // assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
// pHandle->cur.fid = INT32_MIN; // pHandle->cur.fid = INT32_MIN;
// STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
// if (!pCheckInfo->initBuf) { // if (!pCheckInfo->initBuf) {
// initTableMemIterator(pHandle, pCheckInfo); // initTableMemIterator(pHandle, pCheckInfo);
// } // }
...@@ -814,188 +859,154 @@ _err: ...@@ -814,188 +859,154 @@ _err:
// return midSlot; // return midSlot;
// } // }
// static int32_t loadBlockInfo(STsdbReader* pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) { static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, SArray* pIndexList) {
// int32_t code = 0; int32_t code = 0;
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index);
// pCheckInfo->numOfBlocks = 0;
// STable table = {.uid = pCheckInfo->tableId, .suid = pCheckInfo->suid};
// table.pSchema = pTsdbReadHandle->pSchema;
// if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, &table) != TSDB_CODE_SUCCESS) {
// code = terrno;
// return code;
// }
// SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx;
// // no data block in this file, try next file
// if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) {
// return 0; // no data blocks in the file belongs to pCheckInfo->pTable
// }
// if (pCheckInfo->compSize < (int32_t)compIndex->len) {
// assert(compIndex->len > 0);
// char* t = taosMemoryRealloc(pCheckInfo->pCompInfo, compIndex->len);
// if (t == NULL) {
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
// code = TSDB_CODE_TDB_OUT_OF_MEMORY;
// return code;
// }
// pCheckInfo->pCompInfo = (SBlockInfo*)t; SMapData blockIdxMap;
// pCheckInfo->compSize = compIndex->len; tMapDataReset(&blockIdxMap);
// }
// if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { code = tsdbReadBlockIdx(pFileReader, &blockIdxMap, NULL);
// return terrno; if (code != TSDB_CODE_SUCCESS) {
// } goto _err;
// SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; }
// TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; if (blockIdxMap.nItem == 0) {
return TSDB_CODE_SUCCESS;
}
// if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SBlockIdx blockIndex = {0};
// assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && for (int32_t i = 0; i < blockIdxMap.nItem; ++i) {
// pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); code = tMapDataGetItemByIdx(&blockIdxMap, i, &blockIndex, tGetBlockIdx);
// } else { if (code != TSDB_CODE_SUCCESS) {
// assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && goto _err;
// pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); }
// }
// s = TMIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); if (blockIndex.suid != pReader->suid) {
// e = TMAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); continue;
}
// // discard the unqualified data block based on the query time window // this block belongs to a table that is not queried.
// int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); void* p = taosHashGet(pReader->status.pTableMap, &blockIndex.uid, sizeof(uint64_t));
// int32_t end = start; if (p == NULL) {
continue;
}
// if (s > pCompInfo->blocks[start].maxKey.ts) { if ((ASCENDING_TRAVERSE(pReader->order) &&
// return 0; (blockIndex.minKey > pReader->window.ekey || blockIndex.maxKey < pReader->window.skey)) ||
// } (!ASCENDING_TRAVERSE(pReader->order) &&
(blockIndex.minKey > pReader->window.skey || blockIndex.maxKey < pReader->window.ekey))) {
continue;
}
// // todo speedup the procedure of located end block STableBlockScanInfo* pScanInfo = p;
// while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].minKey.ts <= e)) { if (pScanInfo->pBlockList == NULL) {
// end += 1; pScanInfo->pBlockList = taosArrayInit(16, sizeof(SBlock));
// } }
// pCheckInfo->numOfBlocks = (end - start); taosArrayPush(pIndexList, &blockIndex);
}
// if (start > 0) { tMapDataClear(&blockIdxMap);
// memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); return TSDB_CODE_SUCCESS;
// }
// (*numOfBlocks) += pCheckInfo->numOfBlocks; _err:
// return 0; tMapDataClear(&blockIdxMap);
// } return code;
}
// static int32_t getFileCompInfo(STsdbReader* pTsdbReadHandle, int32_t* numOfBlocks) { static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_t* numOfValidTables) {
// // load all the comp offset value for all tables in this file size_t numOfTables = taosArrayGetSize(pIndexList);
// int32_t code = TSDB_CODE_SUCCESS;
// *numOfBlocks = 0;
// pTsdbReadHandle->cost.headFileLoad += 1; *numOfValidTables = 0;
// int64_t s = taosGetTimestampUs();
// size_t numOfTables = 0; for(int32_t i = 0; i < numOfTables; ++i) {
// if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i);
// code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks);
// } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) {
// numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
// for (int32_t i = 0; i < numOfTables; ++i) { SMapData mapData;
// code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks); tMapDataReset(&mapData);
// if (code != TSDB_CODE_SUCCESS) { tsdbReadBlock(pReader->pFileReader, pBlockIdx, &mapData, NULL);
// int64_t e = taosGetTimestampUs();
// pTsdbReadHandle->cost.headFileLoadTime += (e - s); STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t));
// return code; ASSERT(pScanInfo->pBlockList == NULL || taosArrayGetSize(pScanInfo->pBlockList) == 0);
// } for (int32_t j = 0; j < mapData.nItem; ++j) {
// } SBlock block = {0};
// } else {
// assert(0);
// }
// int64_t e = taosGetTimestampUs(); int32_t code = tMapDataGetItemByIdx(&mapData, j, &block, tGetBlock);
// pTsdbReadHandle->cost.headFileLoadTime += (e - s); if (code != TSDB_CODE_SUCCESS) {
// return code; return code;
// } }
// static int32_t doLoadFileDataBlock(STsdbReader* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, if ((ASCENDING_TRAVERSE(pReader->order) &&
// int32_t slotIndex) { (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey)) ||
// int64_t st = taosGetTimestampUs(); (!ASCENDING_TRAVERSE(pReader->order) &&
(block.minKey.ts > pReader->window.skey || block.maxKey.ts < pReader->window.ekey))) {
continue;
}
// int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); void* p = taosArrayPush(pScanInfo->pBlockList, &block);
// if (code != TSDB_CODE_SUCCESS) { if (p == NULL) {
// tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); return TSDB_CODE_OUT_OF_MEMORY;
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; }
// goto _error; }
// }
// code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pTsdbReadHandle->pSchema); if (pScanInfo->pBlockList != NULL && taosArrayGetSize(pScanInfo->pBlockList) > 0) {
// if (code != TSDB_CODE_SUCCESS) { (*numOfValidTables) += 1;
// tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %s", pTsdbReadHandle, pTsdbReadHandle->idStr); }
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; }
// goto _error;
// }
// code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pTsdbReadHandle->pSchema); return TSDB_CODE_SUCCESS;
// if (code != TSDB_CODE_SUCCESS) { }
// tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
// goto _error;
// }
// int16_t* colIds = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) {
int64_t st = taosGetTimestampUs();
int32_t numOfCols = taosArrayGetSize(pReader->pResBlock->pDataBlock);
// int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
// (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)), true); SBlock* pBlock = taosArrayGet(pBlockScanInfo->pBlockList, pFBlock->tbBlockIdx);
// if (ret != TSDB_CODE_SUCCESS) {
// int32_t c = terrno;
// assert(c != TSDB_CODE_SUCCESS);
// goto _error;
// }
// SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo; uint8_t *pb = NULL, *pb1 = NULL;
int32_t code = tsdbReadBlockData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pBlockData, /*pReader->suppInfo.colIdList, numOfCols, */&pb, &pb1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
// pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup; /*
// pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot; int32_t ret = tsdbLoadBlockDataCols(&(pReader->rhelper), pBlock, pCheckInfo->pCompInfo, colIds,
// pBlockLoadInfo->uid = pCheckInfo->tableId; (int)(QH_GET_NUM_OF_COLS(pReader)), true);
if (ret != TSDB_CODE_SUCCESS) {
int32_t c = terrno;
assert(c != TSDB_CODE_SUCCESS);
goto _error;
}
// SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; SDataBlockLoadInfo* pBlockLoadInfo = &pReader->dataBlockLoadInfo;
// assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
// pBlock->numOfRows = pCols->numOfRows; pBlockLoadInfo->fileGroup = pReader->pFileGroup;
pBlockLoadInfo->slot = pReader->cur.slot;
pBlockLoadInfo->uid = pCheckInfo->tableId;
// // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before SDataCols* pCols = pReader->rhelper.pDCols[0];
// 1970-01-01T00:00:00Z if (pBlock->minKey.ts < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
// int64_t* src = pCols->cols[0].pData;
// for (int32_t i = 0; i < pBlock->numOfRows; ++i) {
// src[i] = tdGetKey(src[i]);
// }
// }
// int64_t elapsedTime = (taosGetTimestampUs() - st); pBlock->numOfRows = pCols->numOfRows;
// pTsdbReadHandle->cost.blockLoadTime += elapsedTime; */
// tsdbDebug("%p load file block into buffer, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%" int64_t elapsedTime = (taosGetTimestampUs() - st);
// PRId64 pReader->cost.blockLoadTime += elapsedTime;
// " us, %s",
// pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, elapsedTime,
// pTsdbReadHandle->idStr);
// return TSDB_CODE_SUCCESS;
// _error: tsdbDebug("%p load file block into buffer, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
// pBlock->numOfRows = 0; ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%" PRId64 " us, %s",
pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlockData->nRow,
pBlock->minVersion, pBlock->maxVersion, elapsedTime, pReader->idStr);
return TSDB_CODE_SUCCESS;
// tsdbError("%p error occurs in loading file block, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, %s", _error:
// pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, %s",
// pTsdbReadHandle->idStr); pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->nRow, pReader->idStr);
// return terrno; return code;
// } }
// static int32_t handleDataMergeIfNeeded(STsdbReader* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo) { // static int32_t handleDataMergeIfNeeded(STsdbReader* pTsdbReadHandle, SBlock* pBlock, STableBlockScanInfo* pCheckInfo) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
// STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); // STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
// SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); // SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
...@@ -1041,7 +1052,7 @@ _err: ...@@ -1041,7 +1052,7 @@ _err:
// } // }
// // return error, add test cases // // return error, add test cases
// if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { // if ((code = doLoadFileBlockData(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
// return code; // return code;
// } // }
...@@ -1074,7 +1085,7 @@ _err: ...@@ -1074,7 +1085,7 @@ _err:
// // make sure to only load once // // make sure to only load once
// bool firstTimeExtract = ((cur->pos == 0 && ascScan) || (cur->pos == binfo.rows - 1 && (!ascScan))); // bool firstTimeExtract = ((cur->pos == 0 && ascScan) || (cur->pos == binfo.rows - 1 && (!ascScan)));
// if (pTsdbReadHandle->outputCapacity < binfo.rows && firstTimeExtract) { // if (pTsdbReadHandle->outputCapacity < binfo.rows && firstTimeExtract) {
// code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot); // code = doLoadFileBlockData(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// return code; // return code;
// } // }
...@@ -1102,7 +1113,7 @@ _err: ...@@ -1102,7 +1113,7 @@ _err:
// return code; // return code;
// } // }
// static int32_t loadFileDataBlock(STsdbReader* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, // static int32_t loadFileDataBlock(STsdbReader* pTsdbReadHandle, SBlock* pBlock, STableBlockScanInfo* pCheckInfo,
// bool* exists) { // bool* exists) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
// int32_t code = TSDB_CODE_SUCCESS; // int32_t code = TSDB_CODE_SUCCESS;
...@@ -1111,7 +1122,7 @@ _err: ...@@ -1111,7 +1122,7 @@ _err:
// if (asc) { // if (asc) {
// // query ended in/started from current block // // query ended in/started from current block
// if (pTsdbReadHandle->window.ekey < pBlock->maxKey.ts || pCheckInfo->lastKey > pBlock->minKey.ts) { // if (pTsdbReadHandle->window.ekey < pBlock->maxKey.ts || pCheckInfo->lastKey > pBlock->minKey.ts) {
// if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { // if ((code = doLoadFileBlockData(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
// *exists = false; // *exists = false;
// return code; // return code;
// } // }
...@@ -1135,7 +1146,7 @@ _err: ...@@ -1135,7 +1146,7 @@ _err:
// } // }
// } else { // desc order, query ended in current block // } else { // desc order, query ended in current block
// if (pTsdbReadHandle->window.ekey > pBlock->minKey.ts || pCheckInfo->lastKey < pBlock->maxKey.ts) { // if (pTsdbReadHandle->window.ekey > pBlock->minKey.ts || pCheckInfo->lastKey < pBlock->maxKey.ts) {
// if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { // if ((code = doLoadFileBlockData(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
// *exists = false; // *exists = false;
// return code; // return code;
// } // }
...@@ -1531,7 +1542,7 @@ _err: ...@@ -1531,7 +1542,7 @@ _err:
// } // }
// } // }
// static void updateInfoAfterMerge(STsdbReader* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, // static void updateInfoAfterMerge(STsdbReader* pTsdbReadHandle, STableBlockScanInfo* pCheckInfo, int32_t numOfRows,
// int32_t endPos) { // int32_t endPos) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
...@@ -1562,7 +1573,7 @@ _err: ...@@ -1562,7 +1573,7 @@ _err:
// } // }
// } // }
// static void copyAllRemainRowsFromFileBlock(STsdbReader* pTsdbReadHandle, STableCheckInfo* pCheckInfo, // static void copyAllRemainRowsFromFileBlock(STsdbReader* pTsdbReadHandle, STableBlockScanInfo* pCheckInfo,
// SDataBlockInfo* pBlockInfo, int32_t endPos) { // SDataBlockInfo* pBlockInfo, int32_t endPos) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
...@@ -1648,7 +1659,7 @@ _err: ...@@ -1648,7 +1659,7 @@ _err:
// // only return the qualified data to client in terms of query time window, data rows in the same block but do not // // only return the qualified data to client in terms of query time window, data rows in the same block but do not
// // be included in the query time window will be discarded // // be included in the query time window will be discarded
// static void doMergeTwoLevelData(STsdbReader* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { // static void doMergeTwoLevelData(STsdbReader* pTsdbReadHandle, STableBlockScanInfo* pCheckInfo, SBlock* pBlock) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
// SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); // SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
// STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); // STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
...@@ -1926,180 +1937,164 @@ _err: ...@@ -1926,180 +1937,164 @@ _err:
// return midPos; // return midPos;
// } // }
// static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) {
// taosMemoryFreeClear(pSupporter->numOfBlocksPerTable); taosMemoryFreeClear(pSup->numOfBlocksPerTable);
// taosMemoryFreeClear(pSupporter->blockIndexArray); taosMemoryFreeClear(pSup->indexPerTable);
// for (int32_t i = 0; i < numOfTables; ++i) {
// STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
// taosMemoryFreeClear(pBlockInfo);
// }
// taosMemoryFreeClear(pSupporter->pDataBlockInfo);
// }
// static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
// int32_t leftTableIndex = *(int32_t*)pLeft;
// int32_t rightTableIndex = *(int32_t*)pRight;
// SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param;
// int32_t leftTableBlockIndex = pSupporter->blockIndexArray[leftTableIndex]; for (int32_t i = 0; i < pSup->numOfTables; ++i) {
// int32_t rightTableBlockIndex = pSupporter->blockIndexArray[rightTableIndex]; SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i];
taosMemoryFreeClear(pBlockInfo);
}
// if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftTableIndex]) { taosMemoryFreeClear(pSup->pDataBlockInfo);
// /* left block is empty */ }
// return 1;
// } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightTableIndex]) {
// /* right block is empty */
// return -1;
// }
// STableBlockInfo* pLeftBlockInfoEx = &pSupporter->pDataBlockInfo[leftTableIndex][leftTableBlockIndex]; static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) {
// STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex]; ASSERT(numOfTables >= 1);
// // assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset); pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables);
// #if 0 // TODO: temporarily comment off requested by Dr. Liao pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables);
// if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset && pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables);
// pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
// tsdbError("error in header file, two block with same offset:%" PRId64,
// (int64_t)pLeftBlockInfoEx->compBlock->offset);
// }
// #endif
// return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; if (pSup->numOfBlocksPerTable == NULL || pSup->indexPerTable == NULL || pSup->pDataBlockInfo == NULL) {
// } cleanupBlockOrderSupporter(pSup);
return TSDB_CODE_OUT_OF_MEMORY;
// static int32_t createDataBlocksInfo(STsdbReader* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { }
// size_t size = sizeof(STableBlockInfo) * numOfBlocks;
// if (pTsdbReadHandle->allocSize < size) { return TSDB_CODE_SUCCESS;
// pTsdbReadHandle->allocSize = (int32_t)size; }
// char* tmp = taosMemoryRealloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize);
// if (tmp == NULL) {
// return TSDB_CODE_TDB_OUT_OF_MEMORY;
// }
// pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*)tmp; static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
// } int32_t leftIndex = *(int32_t*)pLeft;
int32_t rightIndex = *(int32_t*)pRight;
// memset(pTsdbReadHandle->pDataBlockInfo, 0, size); SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param;
// *numOfAllocBlocks = numOfBlocks;
// // access data blocks according to the offset of each block in asc/desc order. int32_t leftTableBlockIndex = pSupporter->indexPerTable[leftIndex];
// int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int32_t rightTableBlockIndex = pSupporter->indexPerTable[rightIndex];
// SBlockOrderSupporter sup = {0}; if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftIndex]) {
// sup.numOfTables = numOfTables; /* left block is empty */
// sup.numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); return 1;
// sup.blockIndexArray = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightIndex]) {
// sup.pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); /* right block is empty */
return -1;
}
// if (sup.numOfBlocksPerTable == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { SBlockOrderWrapper* pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex];
// cleanBlockOrderSupporter(&sup, 0); SBlockOrderWrapper* pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex];
// return TSDB_CODE_TDB_OUT_OF_MEMORY;
// }
// int32_t cnt = 0; return pLeftBlock->pBlock->aSubBlock[0].offset > pRightBlock->pBlock->aSubBlock[0].offset ? 1 : -1;
// int32_t numOfQualTables = 0; }
// for (int32_t j = 0; j < numOfTables; ++j) { static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) {
// STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j); pBlockIter->numOfBlocks = numOfBlocks;
// if (pTableCheck->numOfBlocks <= 0) {
// continue;
// }
// SBlock* pBlock = pTableCheck->pCompInfo->blocks; // access data blocks according to the offset of each block in asc/desc order.
// sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap);
// char* buf = taosMemoryMalloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); SBlockOrderSupporter sup = {0};
// if (buf == NULL) { int32_t code = initBlockOrderSupporter(&sup, numOfTables);
// cleanBlockOrderSupporter(&sup, numOfQualTables); if (code != TSDB_CODE_SUCCESS) {
// return TSDB_CODE_TDB_OUT_OF_MEMORY; return code;
// } }
// sup.pDataBlockInfo[numOfQualTables] = (STableBlockInfo*)buf; int32_t cnt = 0;
void* ptr = NULL;
while(1) {
ptr = taosHashIterate(pReader->status.pTableMap, &ptr);
if (ptr == NULL) {
break;
}
// for (int32_t k = 0; k < pTableCheck->numOfBlocks; ++k) { STableBlockScanInfo* pTableScanInfo = (STableBlockScanInfo*)ptr;
// STableBlockInfo* pBlockInfo = &sup.pDataBlockInfo[numOfQualTables][k]; if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) {
continue;
}
// pBlockInfo->compBlock = &pBlock[k]; size_t num = taosArrayGetSize(pTableScanInfo->pBlockList);
// pBlockInfo->pTableCheckInfo = pTableCheck; sup.numOfBlocksPerTable[sup.numOfTables] = num;
// cnt++;
// }
// numOfQualTables++; char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num);
// } if (buf == NULL) {
cleanupBlockOrderSupporter(&sup);
return TSDB_CODE_TDB_OUT_OF_MEMORY;
}
// assert(numOfBlocks == cnt); sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf;
for (int32_t k = 0; k < num; ++k) {
SBlockOrderWrapper wrapper = {0};
wrapper.pBlock = (SBlock*)taosArrayGet(pTableScanInfo->pBlockList, k);
wrapper.uid = pTableScanInfo->uid;
// // since there is only one table qualified, blocks are not sorted sup.pDataBlockInfo[sup.numOfTables][k] = wrapper;
// if (numOfQualTables == 1) { cnt++;
// memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); }
// cleanBlockOrderSupporter(&sup, numOfQualTables);
// tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted %s", pTsdbReadHandle, sup.numOfTables += 1;
// cnt, }
// pTsdbReadHandle->idStr);
// return TSDB_CODE_SUCCESS;
// }
// tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pTsdbReadHandle, cnt, ASSERT(numOfBlocks == cnt);
// numOfQualTables, pTsdbReadHandle->idStr);
// assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 // since there is only one table qualified, blocks are not sorted
// sup.numOfTables = numOfQualTables; if (sup.numOfTables == 1) {
for(int32_t i = 0; i < numOfBlocks; ++i) {
SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i};
taosArrayPush(pBlockIter->blockList, &blockInfo);
}
tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted %s",
pReader, cnt, pReader->idStr);
return TSDB_CODE_SUCCESS;
}
// SMultiwayMergeTreeInfo* pTree = NULL; tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr);
// uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); assert(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0
// if (ret != TSDB_CODE_SUCCESS) {
// cleanBlockOrderSupporter(&sup, numOfTables);
// return TSDB_CODE_TDB_OUT_OF_MEMORY;
// }
// int32_t numOfTotal = 0; SMultiwayMergeTreeInfo* pTree = NULL;
// while (numOfTotal < cnt) { uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar);
// int32_t pos = tMergeTreeGetChosenIndex(pTree); if (ret != TSDB_CODE_SUCCESS) {
// int32_t index = sup.blockIndexArray[pos]++; cleanupBlockOrderSupporter(&sup);
return TSDB_CODE_TDB_OUT_OF_MEMORY;
}
// STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; int32_t numOfTotal = 0;
// pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; while (numOfTotal < cnt) {
int32_t pos = tMergeTreeGetChosenIndex(pTree);
int32_t index = sup.indexPerTable[pos]++;
// // set data block index overflow, in order to disable the offset comparator SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index};
// if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { taosArrayPush(pBlockIter->blockList, &blockInfo);
// sup.blockIndexArray[pos] = sup.numOfBlocksPerTable[pos] + 1;
// }
// tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); // set data block index overflow, in order to disable the offset comparator
// } if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {
sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1;
}
// /* numOfTotal += 1;
// * available when no import exists tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
// * for(int32_t i = 0; i < cnt - 1; ++i) { }
// * assert((*pDataBlockInfo)[i].compBlock->offset < (*pDataBlockInfo)[i+1].compBlock->offset);
// * }
// */
// tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr); tsdbDebug("%p %d data blocks sort completed, %s", pReader, cnt, pReader->idStr);
// cleanBlockOrderSupporter(&sup, numOfTables); cleanupBlockOrderSupporter(&sup);
// taosMemoryFree(pTree); taosMemoryFree(pTree);
// return TSDB_CODE_SUCCESS; pBlockIter->index = 0;
// } return TSDB_CODE_SUCCESS;
}
// static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists); // static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists);
// static int32_t getDataBlock(STsdbReader* pTsdbReadHandle, STableBlockInfo* pNext, bool* exists) { //static int32_t getDataBlock(STsdbReader* pTsdbReadHandle, SFileBlockInfo* pNext, bool* exists) {
// int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; // int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
// SQueryFilePos* cur = &pTsdbReadHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
//
// while (1) { // while (1) {
// int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); // int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
// if (code != TSDB_CODE_SUCCESS || *exists) { // if (code != TSDB_CODE_SUCCESS || *exists) {
// return code; // return code;
// } // }
//
// if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || // if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
// (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
// // all data blocks in current file has been checked already, try next file if exists // // all data blocks in current file has been checked already, try next file if exists
...@@ -2111,7 +2106,7 @@ _err: ...@@ -2111,7 +2106,7 @@ _err:
// pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; // pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
// } // }
// } // }
// } //}
// static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists) { // static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists) {
// pTsdbReadHandle->numOfBlocks = 0; // pTsdbReadHandle->numOfBlocks = 0;
...@@ -2198,7 +2193,7 @@ _err: ...@@ -2198,7 +2193,7 @@ _err:
// cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 0 : pTsdbReadHandle->numOfBlocks - 1; // cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 0 : pTsdbReadHandle->numOfBlocks - 1;
// cur->fid = pTsdbReadHandle->pFileGroup->fid; // cur->fid = pTsdbReadHandle->pFileGroup->fid;
// STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; // SFileBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
// return getDataBlock(pTsdbReadHandle, pBlockInfo, exists); // return getDataBlock(pTsdbReadHandle, pBlockInfo, exists);
// } // }
...@@ -2222,52 +2217,163 @@ _err: ...@@ -2222,52 +2217,163 @@ _err:
// return (numOfRows - startRow) / bucketRange; // return (numOfRows - startRow) / bucketRange;
// } // }
// static int32_t getDataBlocksInFiles(STsdbReader* pTsdbReadHandle, bool* exists) { // query ended in/started from current block
// STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); static int32_t dataBlockPartialRequired(STimeWindow* pWindow, SBlock* pBlock) {
// SQueryFilePos* cur = &pTsdbReadHandle->cur; return ((pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) ||
(pWindow->skey <= pBlock->maxKey.ts && pWindow->skey > pBlock->minKey.ts));
}
// // find the start data block in file static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
// if (!pTsdbReadHandle->locateStart) { SFileDataBlockInfo* pFBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index);
// pTsdbReadHandle->locateStart = true; return pFBlockInfo;
// STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdbReadHandle->pTsdb); }
// int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision);
// tsdbRLockFS(pFileHandle); static bool overlapWithNeighborBlock(SBlock* pBlock, int32_t blockIndex) {
// tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); return false;
// tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); }
// tsdbUnLockFS(pFileHandle);
// return getFirstFileDataBlock(pTsdbReadHandle, exists); static bool bufferDataInFileBlockGap(int32_t order, int64_t key, SBlock* pBlock) {
// } else { bool ascScan = ASCENDING_TRAVERSE(order);
// // check if current file block is all consumed
// STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
// STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
// // current block is done, try next return (ascScan && (key != TSKEY_INITIAL_VAL && key <= pBlock->minKey.ts)) ||
// if ((!cur->mixBlock) || cur->blockCompleted) { (!ascScan && (key != TSKEY_INITIAL_VAL && key >= pBlock->maxKey.ts));
// // all data blocks in current file has been checked already, try next file if exists }
// } else {
// tsdbDebug("%p continue in current data block, index:%d, pos:%d, %s", pTsdbReadHandle, cur->slot, cur->pos,
// pTsdbReadHandle->idStr);
// int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo);
// *exists = (pTsdbReadHandle->realNumOfRows > 0);
// if (code != TSDB_CODE_SUCCESS || *exists) { static int32_t buildInmemDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlock* pBlock, TSDBKEY *key) {
// return code; int32_t code = TSDB_CODE_SUCCESS;
// }
// }
// // current block is empty, try next block in file bool ascScan = ASCENDING_TRAVERSE(pReader->order);
// // all data blocks in current file has been checked already, try next file if exists bool cacheDataInFileBlockHole = (ascScan && (key->ts != TSKEY_INITIAL_VAL && key->ts < pBlock->minKey.ts)) ||
// if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { (!ascScan && (key->ts != TSKEY_INITIAL_VAL && key->ts > pBlock->maxKey.ts));
// return getFirstFileDataBlock(pTsdbReadHandle, exists); ASSERT(cacheDataInFileBlockHole);
// } else {
// moveToNextDataBlockInCurrentFile(pTsdbReadHandle); // do not load file block into buffer
// STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; int32_t step = ascScan ? 1 : -1;
// return getDataBlock(pTsdbReadHandle, pNext, exists);
// } TSDBKEY maxKey = {.version = pReader->endVersion};
// } maxKey.ts = ascScan ? (pBlock->minKey.ts - step) : (pBlock->maxKey.ts - step);
// }
pBlockScanInfo->memHasVal = tsdbTbDataIterNext(&pBlockScanInfo->iter);
pBlockScanInfo->imemHasVal = tsdbTbDataIterNext(&pBlockScanInfo->iiter);
code = tsdbReadRowsFromCache(pBlockScanInfo, maxKey, pReader->capacity, pReader);
return code;
}
static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
SFileBlockDumpInfo *pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pData = &pReader->status.fileBlockData;
STSchema* pSchema = NULL;
SRowMerger merge = {0};
int64_t key = pData->aTSKEY[0];
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader);
if (pBlockScanInfo->memHasVal && pBlockScanInfo->imemHasVal) {
TSDBKEY k = TSDBROW_KEY(pRow);
TSDBKEY ik = TSDBROW_KEY(piRow);
// todo check version in file
if (key < k.ts || key < ik.ts) {
tRowMergerInit(&merge, NULL, pSchema);
STSRow* pTsRow = NULL;
doLoadRowsOfIdenticalTsInFileBlock(pData, pDumpInfo, key, &merge, pReader, &pTsRow);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) {
SReaderStatus* pStatus = &pReader->status;
SFileSetIter* pFIter = &pStatus->fileIter;
if (pFIter->index < pFIter->numOfFiles) {
if (pReader->status.blockIter.index == -1) {
int32_t numOfBlocks = 0;
while (1) {
bool hasNext = nextFilesetIterator(&pStatus->fileIter, pReader->order, pReader);
if (!hasNext) {
// no data files on disk
break;
}
SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx));
int32_t code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (taosArrayGetSize(pIndexList) > 0) {
uint32_t numOfValidTable = 0;
code = doLoadFileBlock(pReader, pIndexList, &numOfValidTable);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (numOfValidTable > 0) {
break;
}
}
// no blocks in current file, try next files
}
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
int32_t code = initBlockIterator(pReader, pBlockIter, numOfBlocks);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
int64_t key = 0; // todo get the first qualified key in buffer
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx);
if (pScanInfo->iterInit == false) {
STbData* d = NULL;
tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pScanInfo->uid, &d);
TSDBKEY startKey = {.ts = pReader->window.skey, .version = pReader->startVersion};
tsdbTbDataIterOpen(d, &startKey, 0, &pScanInfo->iter);
STbData* di = NULL;
tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pScanInfo->uid, &di);
tsdbTbDataIterOpen(di, &startKey, 0, &pScanInfo->iiter);
pScanInfo->iterInit = true;
}
if (dataBlockPartialRequired(&pReader->window, pBlock) || overlapWithNeighborBlock(pBlock, pFBlock->tbBlockIdx)) {
SBlockData data = {0};
doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &data);
// build composed data block
} else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) {
// data in memory that are earlier than current file block
TSDBKEY maxKey = {.ts = pReader->window.ekey, .version = pReader->endVersion};
buildInmemDataBlock(pReader, pScanInfo, pBlock, &maxKey);
// build data block from in-memory buffer data completed.
} else { // whole block is required, return it directly
SDataBlockInfo info = {0};
info.rows = pBlock->nRow;
info.uid = pScanInfo->uid;
info.window.skey = pBlock->minKey.ts;
info.window.ekey = pBlock->maxKey.ts;
}
} else {
}
}
return TSDB_CODE_SUCCESS;
}
// static bool doHasDataInBuffer(STsdbReader* pTsdbReadHandle) { // static bool doHasDataInBuffer(STsdbReader* pTsdbReadHandle) {
// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); // size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
...@@ -2293,7 +2399,7 @@ _err: ...@@ -2293,7 +2399,7 @@ _err:
// int32_t i = 0; // int32_t i = 0;
// while (i < numOfTables) { // while (i < numOfTables) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
// // the first qualified table for interpolation query // // the first qualified table for interpolation query
// // if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && // // if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) &&
...@@ -2309,71 +2415,188 @@ _err: ...@@ -2309,71 +2415,188 @@ _err:
// return; // return;
// } // }
// STableCheckInfo info = *(STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // STableBlockScanInfo info = *(STableBlockScanInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
// taosArrayClear(pTsdbReadHandle->pTableCheckInfo); // taosArrayClear(pTsdbReadHandle->pTableCheckInfo);
// info.lastKey = pTsdbReadHandle->window.skey; // info.lastKey = pTsdbReadHandle->window.skey;
// taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); // taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info);
// } // }
// static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader) {
// STsdbReader* pTsdbReadHandle) { if (!hasVal) {
// int numOfRows = 0; return NULL;
// int curRows = 0; }
// int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
// STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
// win->skey = TSKEY_INITIAL_VAL;
// int64_t st = taosGetTimestampUs(); TSDBROW* pRow = tsdbTbDataIterGet(pIter);
// int16_t rv = -1;
// STSchema* pSchema = NULL;
// TSKEY lastRowKey = TSKEY_INITIAL_VAL;
// do { TSDBKEY key = TSDBROW_KEY(pRow);
// STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX); if (key.ts > pReader->window.ekey) {
// if (row == NULL) { *hasVal = false;
// break; return NULL;
// } }
// TSKEY key = TD_ROW_KEY(row); if (key.version <= pReader->endVersion) {
// if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || return pRow;
// (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { }
// tsdbDebug("%p key:%" PRIu64 " beyond qrange:%" PRId64 " - %" PRId64 ", no more data in buffer",
// pTsdbReadHandle,
// key, pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey);
// break; while(1) {
// } *hasVal = tsdbTbDataIterNext(pIter);
if (!(*hasVal)) {
return NULL;
}
// if (win->skey == INT64_MIN) { pRow = tsdbTbDataIterGet(pIter);
// win->skey = key;
// }
// win->ekey = key; key = TSDBROW_KEY(pRow);
// if (rv != TD_ROW_SVER(row)) { if (key.ts > pReader->window.ekey) {
// pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, TD_ROW_SVER(row)); *hasVal = false;
// rv = TD_ROW_SVER(row); return NULL;
// } }
// numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols,
// pCheckInfo->tableId,
// pSchema, NULL, pCfg->update, &lastRowKey);
// if (numOfRows >= maxRowsToRead) { if (key.version <= pReader->endVersion) {
// moveToNextRowInMem(pCheckInfo); return pRow;
// break; }
// } }
}
// } while (moveToNextRowInMem(pCheckInfo)); int32_t doLoadRowsOfIdenticalTs(STbDataIter *pIter, bool* hasVal, int64_t ts, SRowMerger* pMerger, STsdbReader* pReader) {
while (1) {
*hasVal = tsdbTbDataIterNext(pIter);
if (!*hasVal) {
break;
}
// taosMemoryFreeClear(pSchema); // free the STSChema TSDBROW* pRow = getValidRow(pIter, hasVal, pReader);
// assert(numOfRows <= maxRowsToRead); TSDBKEY k = TSDBROW_KEY(pRow);
if (k.ts > ts) {
break;
}
// int64_t elapsedTime = taosGetTimestampUs() - st; tRowMerge(pMerger, pRow);
// tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s", }
// pTsdbReadHandle, elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr);
// return numOfRows; return TSDB_CODE_SUCCESS;
// } }
int32_t doLoadRowsOfIdenticalTsInFileBlock(SBlockData* pData, SFileBlockDumpInfo* pDumpInfo, int64_t ts, SRowMerger *pMerger,
STsdbReader* pReader, STSRow** pRow) {
int64_t key = pData->aTSKEY[pDumpInfo->rowIndex];
if ((pDumpInfo->rowIndex < pData->nRow - 1)) {
if (pData->aTSKEY[pDumpInfo->rowIndex + 1] < key) {
SRowMerger merger = {0};
tRowMergerInit(&merger, NULL, NULL);
int32_t rowIndex = pDumpInfo->rowIndex + 1;
while (pData->aTSKEY[rowIndex] == key) {
tRowMerge(&merger, NULL);
}
tRowMergerGetRow(&merger, pRow);
tRowMergerClear(&merger);
}
} else {
}
return TSDB_CODE_SUCCESS;
}
int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow) {
STSchema* pSchema = NULL; // todo set the correct schema
TSKEY mergeTs = TSKEY_INITIAL_VAL;
SRowMerger merge = {0};
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader);
TSDBKEY k = {.ts = TSKEY_INITIAL_VAL};
TSDBKEY ik = {.ts = TSKEY_INITIAL_VAL};
if (pBlockScanInfo->memHasVal && pBlockScanInfo->imemHasVal) {
k = TSDBROW_KEY(pRow);
ik = TSDBROW_KEY(piRow);
if (ik.ts <= k.ts) {
tRowMergerInit(&merge, piRow, pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
if (k.ts == mergeTs) {
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
}
tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS;
} else { // k.ts < ik.ts
tRowMergerInit(&merge, pRow, pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS;
}
}
if (pBlockScanInfo->memHasVal) {
tRowMergerInit(&merge, pRow, pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS;
}
if (pBlockScanInfo->imemHasVal) {
tRowMergerInit(&merge, piRow, pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_SUCCESS;
}
int32_t tsdbReadRowsFromCache(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader) {
int32_t numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pReader->pResBlock->pDataBlock);
SSDataBlock* pBlock = pReader->pResBlock;
int64_t st = taosGetTimestampUs();
STSchema* pSchema = NULL;
do {
STSRow* pTSRow = NULL;
tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow);
// todo assign to ssdatablock
if (pBlockScanInfo->memHasVal) {
TSDBROW* pRow = tsdbTbDataIterGet(&pBlockScanInfo->iter);
TSDBKEY k = TSDBROW_KEY(pRow);
if (k.ts >= maxKey.ts) {
break;
}
}
if (pBlockScanInfo->imemHasVal) {
TSDBROW* pRow = tsdbTbDataIterGet(&pBlockScanInfo->iiter);
TSDBKEY k = TSDBROW_KEY(pRow);
if (k.ts >= maxKey.ts) {
break;
}
}
// no data in buffer, return immediately
if (!(pBlockScanInfo->memHasVal || pBlockScanInfo->imemHasVal)) {
break;
}
if (numOfRows >= capacity) {
break;
}
} while (1);
taosMemoryFreeClear(pSchema);
assert(numOfRows <= capacity);
int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s",
pReader, elapsedTime, pBlock->info.rows, numOfCols, pReader->idStr);
return TSDB_CODE_SUCCESS;
}
// static void destroyHelper(void* param) { // static void destroyHelper(void* param) {
// if (param == NULL) { // if (param == NULL) {
...@@ -2398,7 +2621,7 @@ _err: ...@@ -2398,7 +2621,7 @@ _err:
// // check if the query range overlaps with the file data block // // check if the query range overlaps with the file data block
// bool exists = true; // bool exists = true;
// int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); // int32_t code = loadDataInFiles(pTsdbReadHandle, &exists);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// pTsdbReadHandle->checkFiles = false; // pTsdbReadHandle->checkFiles = false;
// return false; // return false;
...@@ -2459,7 +2682,7 @@ _err: ...@@ -2459,7 +2682,7 @@ _err:
// int32_t curRow = 0; // int32_t curRow = 0;
// if (++pTsdbReadHandle->activeIndex < numOfTables) { // if (++pTsdbReadHandle->activeIndex < numOfTables) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
// // int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); // // int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key);
// // if (ret != TSDB_CODE_SUCCESS) { // // if (ret != TSDB_CODE_SUCCESS) {
// // return false; // // return false;
...@@ -2494,7 +2717,7 @@ _err: ...@@ -2494,7 +2717,7 @@ _err:
// return true; // return true;
// } // }
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
// pCheckInfo->numOfBlocks = 0; // pCheckInfo->numOfBlocks = 0;
// pTsdbReadHandle->activeIndex += 1; // pTsdbReadHandle->activeIndex += 1;
...@@ -2543,17 +2766,6 @@ _err: ...@@ -2543,17 +2766,6 @@ _err:
// return true; // return true;
// } // }
// static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
// if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) {
// return -1;
// } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) {
// return 1;
// } else {
// ASSERT(false);
// return 0;
// }
// }
// static void* doFreeColumnInfoData(SArray* pColumnInfoData) { // static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
// if (pColumnInfoData == NULL) { // if (pColumnInfoData == NULL) {
// return NULL; // return NULL;
...@@ -2572,7 +2784,7 @@ _err: ...@@ -2572,7 +2784,7 @@ _err:
// static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { // static void* destroyTableCheckInfo(SArray* pTableCheckInfo) {
// size_t size = taosArrayGetSize(pTableCheckInfo); // size_t size = taosArrayGetSize(pTableCheckInfo);
// for (int32_t i = 0; i < size; ++i) { // for (int32_t i = 0; i < size; ++i) {
// STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); // STableBlockScanInfo* p = taosArrayGet(pTableCheckInfo, i);
// destroyTableMemIterator(p); // destroyTableMemIterator(p);
// taosMemoryFreeClear(p->pCompInfo); // taosMemoryFreeClear(p->pCompInfo);
...@@ -2583,25 +2795,31 @@ _err: ...@@ -2583,25 +2795,31 @@ _err:
// } // }
// ====================================== EXPOSED APIs ====================================== // ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* pTableList, uint64_t qId, uint64_t taskId, STsdbReader** ppReader) {
uint64_t taskId, STsdbReader** ppReader) { char buf[128] = {0};
int32_t code = 0; snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId);
code = tsdbReaderCreate(pVnode, pCond, qId, taskId, ppReader); int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, buf);
if (code) goto _err; if (code) {
goto _err;
}
// if (emptyQueryTimewindow(pReader)) { STsdbReader* pReader = *ppReader;
// return (STsdbReader*)pReader; if (isEmptyQueryTimeWindow(pReader)) {
// } tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr);
return TSDB_CODE_SUCCESS;
}
// // todo apply the lastkey of table check to avoid to load header file pReader->status.pTableMap = createDataBlockScanInfo(pReader, pCond->uidList, pCond->numOfTables);
// pReader->pTableCheckInfo = createCheckInfoFromTableGroup(pReader, tableList); if (pReader->status.pTableMap == NULL) {
// if (pReader->pTableCheckInfo == NULL) { tsdbReaderClose(pReader);
// // tsdbReaderClose(pTsdbReadHandle); *ppReader = NULL;
// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
// return NULL; code = TSDB_CODE_TDB_OUT_OF_MEMORY;
// } goto _err;
}
#if 0
// int32_t code = setCurrentSchema(pVnode, pReader); // int32_t code = setCurrentSchema(pVnode, pReader);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// terrno = code; // terrno = code;
...@@ -2627,94 +2845,90 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInf ...@@ -2627,94 +2845,90 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInf
// return NULL; // return NULL;
// } // }
// } // }
#endif
// tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pReader, tsdbDebug("%p total numOfTable:%d in this query %s", pReader, pCond->numOfTables, pReader->idStr);
// taosArrayGetSize(pReader->pTableCheckInfo), taosArrayGetSize(tableList->pTableList), pReader->idStr);
return code; return code;
_err: _err:
// tsdbError(""); tsdbError("failed to create tsdb reader, code: %s %s", tstrerror(code), pReader->idStr);
return code; return code;
} }
void tsdbReaderClose(STsdbReader* pReader) { void tsdbReaderClose(STsdbReader* pReader) {
// if (pReader == NULL) { if (pReader == NULL) {
// return; return;
// } }
// pReader->pColumns = doFreeColumnInfoData(pReader->pColumns);
// taosArrayDestroy(pReader->suppInfo.defaultLoadColumn);
// taosMemoryFreeClear(pReader->pDataBlockInfo);
// taosMemoryFreeClear(pReader->suppInfo.pstatis);
// taosMemoryFreeClear(pReader->suppInfo.plist);
// taosMemoryFree(pReader->suppInfo.slotIds);
// if (!emptyQueryTimewindow(pReader)) { blockDataDestroy(pReader->pResBlock);
// // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
// } else {
// assert(pReader->pTableCheckInfo == NULL);
// }
// if (pReader->pTableCheckInfo != NULL) { taosMemoryFreeClear(pReader->suppInfo.pstatis);
// pReader->pTableCheckInfo = destroyTableCheckInfo(pReader->pTableCheckInfo); taosMemoryFreeClear(pReader->suppInfo.plist);
// } taosMemoryFree(pReader->suppInfo.slotIds);
// tsdbDestroyReadH(&pReader->rhelper); if (!isEmptyQueryTimeWindow(pReader)) {
// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
} else {
ASSERT(pReader->status.pTableMap == NULL);
}
#if 0
// if (pReader->status.pTableScanInfo != NULL) {
// pReader->status.pTableScanInfo = destroyTableCheckInfo(pReader->status.pTableScanInfo);
// }
// tdFreeDataCols(pReader->pDataCols); // tsdbDestroyReadH(&pReader->rhelper);
// pReader->pDataCols = NULL;
// pReader->prev = doFreeColumnInfoData(pReader->prev); // tdFreeDataCols(pReader->pDataCols);
// pReader->next = doFreeColumnInfoData(pReader->next); // pReader->pDataCols = NULL;
//
// pReader->prev = doFreeColumnInfoData(pReader->prev);
// pReader->next = doFreeColumnInfoData(pReader->next);
#endif
// SIOCostSummary* pCost = &pReader->cost; SIOCostSummary* pCost = &pReader->cost;
// tsdbDebug("%p :io-cost summary: head-file read cnt:%" PRIu64 ", head-file time:%" PRIu64 " us, statis-info:%" tsdbDebug("%p :io-cost summary: head-file read cnt:%" PRIu64 ", head-file time:%" PRIu64 " us, statis-info:%" PRId64
// PRId64 " us, datablock:%" PRId64 " us, check data:%" PRId64 " us, %s",
// " us, datablock:%" PRId64 " us, check data:%" PRId64 " us, %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime,
// pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pReader->idStr);
// pCost->checkForNextTime, pReader->idStr);
// taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->idStr);
// taosMemoryFree(pReader->pSchema); taosMemoryFree(pReader->pSchema);
// taosMemoryFreeClear(pReader); taosMemoryFreeClear(pReader);
} }
bool tsdbNextDataBlock(STsdbReader* pReader) { bool tsdbNextDataBlock(STsdbReader* pReader) {
bool ret = false; bool ret = false;
// size_t numOfCols = taosArrayGetSize(pReader->pColumns); if (isEmptyQueryTimeWindow(pReader)) {
// for (int32_t i = 0; i < numOfCols; ++i) { return false;
// SColumnInfoData* pColInfo = taosArrayGet(pReader->pColumns, i); }
// colInfoDataCleanup(pColInfo, pReader->outputCapacity);
// }
// if (emptyQueryTimewindow(pReader)) { // cleanup the data that belongs to the previous data block
// tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); blockDataCleanup(pReader->pResBlock);
// return false;
// }
// int64_t stime = taosGetTimestampUs(); int64_t stime = taosGetTimestampUs();
// int64_t elapsedTime = stime; int64_t elapsedTime = stime;
// // TODO refactor: remove "type" if (pReader->type == BLOCK_LOAD_OFFSET_ORDER) {
// if (pReader->type == TSDB_QUERY_TYPE_LAST) { if (pReader->status.loadFromFile) {
// if (pReader->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { bool exists = true;
// // return loadCachedLastRow(pTsdbReadHandle); int32_t code = loadDataInFiles(pReader, &exists);
// } else if (pReader->cachelastrow == TSDB_CACHED_TYPE_LAST) {
// // return loadCachedLast(pTsdbReadHandle);
// }
// }
} else { // no data in files, let's try in-memory buffer
}
} else if (pReader->type == BLOCK_LOAD_TABLESEQ_ORDER) {
} else if (pReader->type == BLOCK_LOAD_EXTERN_ORDER) {
}
// if (pReader->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { // if (pReader->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
// return loadDataBlockFromTableSeq(pReader); // return loadDataBlockFromTableSeq(pReader);
// } else { // loadType == RR and Offset Order // } else { // loadType == RR and Offset Order
// if (pReader->checkFiles) { // if (pReader->checkFiles) {
// // check if the query range overlaps with the file data block // // check if the query range overlaps with the file data block
// bool exists = true; // bool exists = true;
// int32_t code = loadDataInFiles(pReader, &exists);
// int32_t code = getDataBlocksInFiles(pReader, &exists);
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// pReader->activeIndex = 0; // pReader->activeIndex = 0;
// pReader->checkFiles = false; // pReader->checkFiles = false;
...@@ -2749,10 +2963,10 @@ void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockI ...@@ -2749,10 +2963,10 @@ void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockI
// // there are data in file // // there are data in file
// if (pReader->cur.fid != INT32_MIN) { // if (pReader->cur.fid != INT32_MIN) {
// STableBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[cur->slot]; // SFileBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[cur->slot];
// uid = pBlockInfo->pTableCheckInfo->tableId; // uid = pBlockInfo->pTableCheckInfo->tableId;
// } else { // } else {
// STableCheckInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, pReader->activeIndex); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, pReader->activeIndex);
// uid = pCheckInfo->tableId; // uid = pCheckInfo->tableId;
// } // }
...@@ -2781,7 +2995,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** ...@@ -2781,7 +2995,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg***
// return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
// } // }
// STableBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[c->slot]; // SFileBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[c->slot];
// assert((c->slot >= 0 && c->slot < pReader->numOfBlocks) || ((c->slot == pReader->numOfBlocks) && (c->slot == 0))); // assert((c->slot >= 0 && c->slot < pReader->numOfBlocks) || ((c->slot == pReader->numOfBlocks) && (c->slot == 0)));
// // file block with sub-blocks has no statistics data // // file block with sub-blocks has no statistics data
...@@ -2854,8 +3068,8 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { ...@@ -2854,8 +3068,8 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
// if (pReader->cur.fid == INT32_MIN) { // if (pReader->cur.fid == INT32_MIN) {
// return pReader->pColumns; // return pReader->pColumns;
// } else { // } else {
// STableBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[pReader->cur.slot]; // SFileBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[pReader->cur.slot];
// STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // STableBlockScanInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
// if (pReader->cur.mixBlock) { // if (pReader->cur.mixBlock) {
// return pReader->pColumns; // return pReader->pColumns;
...@@ -2871,7 +3085,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { ...@@ -2871,7 +3085,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
// return pReader->pColumns; // return pReader->pColumns;
// } else { // only load the file block // } else { // only load the file block
// SBlock* pBlock = pBlockInfo->compBlock; // SBlock* pBlock = pBlockInfo->compBlock;
// if (doLoadFileDataBlock(pReader, pBlock, pCheckInfo, pReader->cur.slot) != TSDB_CODE_SUCCESS) { // if (doLoadFileBlockData(pReader, pBlock, pCheckInfo, pReader->cur.slot) != TSDB_CODE_SUCCESS) {
// return NULL; // return NULL;
// } // }
...@@ -2884,7 +3098,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { ...@@ -2884,7 +3098,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
} }
void tsdbResetReadHandle(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) { void tsdbResetReadHandle(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) {
// if (emptyQueryTimewindow(pReader)) { // if (isEmptyQueryTimeWindow(pReader)) {
// if (pCond->order != pReader->order) { // if (pCond->order != pReader->order) {
// pReader->order = pCond->order; // pReader->order = pCond->order;
// TSWAP(pReader->window.skey, pReader->window.ekey); // TSWAP(pReader->window.skey, pReader->window.ekey);
...@@ -2998,7 +3212,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa ...@@ -2998,7 +3212,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
// pTableBlockInfo->numOfBlocks += numOfBlocks; // pTableBlockInfo->numOfBlocks += numOfBlocks;
// for (int32_t i = 0; i < numOfTables; ++i) { // for (int32_t i = 0; i < numOfTables; ++i) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, i); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, i);
// SBlock* pBlock = pCheckInfo->pCompInfo->blocks; // SBlock* pBlock = pCheckInfo->pCompInfo->blocks;
...@@ -3040,7 +3254,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { ...@@ -3040,7 +3254,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) {
// size_t size = taosArrayGetSize(pReader->pTableCheckInfo); // size_t size = taosArrayGetSize(pReader->pTableCheckInfo);
// for (int32_t i = 0; i < size; ++i) { // for (int32_t i = 0; i < size; ++i) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, i); // STableBlockScanInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, i);
// // if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { // // if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
// // pMem = pMemT->tData[pCheckInfo->tableId]; // // pMem = pMemT->tData[pCheckInfo->tableId];
......
source/libs/executor/src/executil.c
浏览文件 @ dbd8c300
...@@ -620,8 +620,6 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { ...@@ -620,8 +620,6 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
} }
int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
pCond->loadExternalRows = false;
pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
...@@ -647,15 +645,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi ...@@ -647,15 +645,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
} }
#endif #endif
for (int32_t i = 0; i < pCond->numOfTWindows; ++i) { pCond->type = BLOCK_LOAD_OFFSET_ORDER;
if ((pCond->order == TSDB_ORDER_ASC && pCond->twindows[i].skey > pCond->twindows[i].ekey) ||
(pCond->order == TSDB_ORDER_DESC && pCond->twindows[i].skey < pCond->twindows[i].ekey)) {
TSWAP(pCond->twindows[i].skey, pCond->twindows[i].ekey);
}
}
taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow);
pCond->type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
// pCond->type = pTableScanNode->scanFlag; // pCond->type = pTableScanNode->scanFlag;
int32_t j = 0; int32_t j = 0;
...@@ -677,6 +667,5 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi ...@@ -677,6 +667,5 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
} }
void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) {
taosMemoryFree(pCond->twindows);
taosMemoryFree(pCond->colList); taosMemoryFree(pCond->colList);
} }
\ No newline at end of file
source/libs/executor/src/executorimpl.c
浏览文件 @ dbd8c300
...@@ -4306,7 +4306,11 @@ STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle ...@@ -4306,7 +4306,11 @@ STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle
} }
STsdbReader* pReader; STsdbReader* pReader;
tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId, &pReader); code = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId, &pReader);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
cleanupQueryTableDataCond(&cond); cleanupQueryTableDataCond(&cond);
return pReader; return pReader;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或