提交 14fe658f 编写于 作者: H hjxilinx

refactor some codes, remove the mmap of header file for super table query. issue #969 [tbase-1368]

上级 5741c026
...@@ -79,7 +79,7 @@ extern "C" { ...@@ -79,7 +79,7 @@ extern "C" {
{ \ { \
if (FD_VALID(x)) { \ if (FD_VALID(x)) { \
close(x); \ close(x); \
x = -1; \ x = FD_INITIALIZER; \
} \ } \
} }
......
...@@ -180,8 +180,8 @@ void queryOnBlock(SMeterQuerySupportObj* pSupporter, int64_t* primaryKeys, int32 ...@@ -180,8 +180,8 @@ void queryOnBlock(SMeterQuerySupportObj* pSupporter, int64_t* primaryKeys, int32
SBlockInfo* pBlockBasicInfo, SMeterDataInfo* pDataHeadInfoEx, SField* pFields, SBlockInfo* pBlockBasicInfo, SMeterDataInfo* pDataHeadInfoEx, SField* pFields,
__block_search_fn_t searchFn); __block_search_fn_t searchFn);
SMeterDataInfo** vnodeFilterQualifiedMeters(SQInfo* pQInfo, int32_t vid, int32_t fileIndex, int32_t vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, tSidSet *pSidSet, SMeterDataInfo *pMeterDataInfo,
tSidSet* pSidSet, SMeterDataInfo* pMeterDataInfo, int32_t* numOfMeters); int32_t *numOfMeters, SMeterDataInfo ***pReqMeterDataInfo);
int32_t vnodeGetVnodeHeaderFileIdx(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order); int32_t vnodeGetVnodeHeaderFileIdx(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters, int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters,
...@@ -196,11 +196,11 @@ int32_t setIntervalQueryExecutionContext(SMeterQuerySupportObj* pSupporter, int3 ...@@ -196,11 +196,11 @@ int32_t setIntervalQueryExecutionContext(SMeterQuerySupportObj* pSupporter, int3
int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange); int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange);
int64_t getNextAccessedKeyInData(SQuery* pQuery, int64_t* pPrimaryCol, SBlockInfo* pBlockInfo, int32_t blockStatus); int64_t getNextAccessedKeyInData(SQuery* pQuery, int64_t* pPrimaryCol, SBlockInfo* pBlockInfo, int32_t blockStatus);
uint32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, char* pHeaderData, int32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters,
int32_t numOfMeters, const char* filePath, SMeterDataInfo** pMeterDataInfo); const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks);
int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv, int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv,
int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand); int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand);
char *vnodeGetHeaderFileData(SQueryRuntimeEnv *pRuntimeEnv, int32_t vnodeId, int32_t fileIndex); int32_t vnodeGetHeaderFile(SQueryRuntimeEnv *pRuntimeEnv, int32_t fileIndex);
/** /**
* Create SMeterQueryInfo. * Create SMeterQueryInfo.
......
...@@ -100,12 +100,11 @@ typedef struct SQueryFilesInfo { ...@@ -100,12 +100,11 @@ typedef struct SQueryFilesInfo {
uint32_t numOfFiles; // the total available number of files for this virtual node during query execution uint32_t numOfFiles; // the total available number of files for this virtual node during query execution
int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap.
int32_t vnodeId; int32_t vnodeId;
int32_t headerFd; // header file fd int32_t headerFd; // header file fd
char* pHeaderFileData; // mmap header files int64_t headerFileSize;
int64_t headFileSize; int32_t dataFd;
int32_t dataFd; int32_t lastFd;
int32_t lastFd;
char headerFilePath[PATH_MAX]; // current opened header file name char headerFilePath[PATH_MAX]; // current opened header file name
char dataFilePath[PATH_MAX]; // current opened data file name char dataFilePath[PATH_MAX]; // current opened data file name
...@@ -165,11 +164,10 @@ typedef struct SMeterDataInfo { ...@@ -165,11 +164,10 @@ typedef struct SMeterDataInfo {
uint64_t offsetInHeaderFile; uint64_t offsetInHeaderFile;
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t start; // start block index int32_t start; // start block index
SCompBlock** pBlock; SCompBlock* pBlock;
int32_t meterOrderIdx; int32_t meterOrderIdx;
SMeterObj* pMeterObj; SMeterObj* pMeterObj;
int32_t groupIdx; // group id in meter list int32_t groupIdx; // group id in meter list
SMeterQueryInfo* pMeterQInfo; SMeterQueryInfo* pMeterQInfo;
} SMeterDataInfo; } SMeterDataInfo;
......
...@@ -295,8 +295,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe ...@@ -295,8 +295,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
pQuery->fileId = fid; pQuery->fileId = fid;
pSummary->numOfFiles++; pSummary->numOfFiles++;
char *pHeaderFileData = vnodeGetHeaderFileData(pRuntimeEnv, vnodeId, fileIdx); if (vnodeGetHeaderFile(pRuntimeEnv, fileIdx) != TSDB_CODE_SUCCESS) {
if (pHeaderFileData == NULL) { // failed to mmap header file into buffer, ignore current file, try next
fid += step; fid += step;
continue; continue;
} }
...@@ -304,14 +303,16 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe ...@@ -304,14 +303,16 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
int32_t numOfQualifiedMeters = 0; int32_t numOfQualifiedMeters = 0;
assert(fileIdx == pRuntimeEnv->vnodeFileInfo.current); assert(fileIdx == pRuntimeEnv->vnodeFileInfo.current);
SMeterDataInfo **pReqMeterDataInfo = vnodeFilterQualifiedMeters(pQInfo, vnodeId, fileIdx, pSupporter->pSidSet, SMeterDataInfo **pReqMeterDataInfo = NULL;
pMeterDataInfo, &numOfQualifiedMeters); int32_t ret = vnodeFilterQualifiedMeters(pQInfo, vnodeId, pSupporter->pSidSet, pMeterDataInfo,
&numOfQualifiedMeters, &pReqMeterDataInfo);
if (pReqMeterDataInfo == NULL) { if (ret != TSDB_CODE_SUCCESS) {
dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo); dError("QInfo:%p failed to create meterdata struct to perform query processing, abort", pQInfo);
pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; tfree(pReqMeterDataInfo);
pQInfo->code = -ret;
pQInfo->killed = 1; pQInfo->killed = 1;
return NULL; return NULL;
} }
...@@ -324,8 +325,18 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe ...@@ -324,8 +325,18 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
continue; continue;
} }
uint32_t numOfBlocks = getDataBlocksForMeters(pSupporter, pQuery, pHeaderFileData, numOfQualifiedMeters, uint32_t numOfBlocks = 0;
pVnodeFileInfo->headerFilePath, pReqMeterDataInfo); ret = getDataBlocksForMeters(pSupporter, pQuery, numOfQualifiedMeters, pVnodeFileInfo->headerFilePath,
pReqMeterDataInfo, &numOfBlocks);
if (ret != TSDB_CODE_SUCCESS) {
dError("QInfo:%p failed to get data block before scan data blocks, abort", pQInfo);
tfree(pReqMeterDataInfo);
pQInfo->code = -ret;
pQInfo->killed = 1;
return NULL;
}
dTrace("QInfo:%p file:%s, %d meters contains %d blocks to be checked", pQInfo, pVnodeFileInfo->dataFilePath, dTrace("QInfo:%p file:%s, %d meters contains %d blocks to be checked", pQInfo, pVnodeFileInfo->dataFilePath,
numOfQualifiedMeters, numOfBlocks); numOfQualifiedMeters, numOfBlocks);
...@@ -336,13 +347,13 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe ...@@ -336,13 +347,13 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
continue; continue;
} }
int32_t n = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks, ret = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks,
&nAllocBlocksInfoSize, (int64_t)pQInfo); &nAllocBlocksInfoSize, (int64_t)pQInfo);
if (n < 0) { // failed to create data blocks if (ret != TSDB_CODE_SUCCESS) { // failed to create data blocks
dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo); dError("QInfo:%p build blockInfoEx failed, abort", pQInfo);
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; pQInfo->code = -ret;
pQInfo->killed = 1; pQInfo->killed = 1;
return NULL; return NULL;
} }
...@@ -397,7 +408,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe ...@@ -397,7 +408,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
setExecutionContext(pSupporter, pSupporter->pResult, pOneMeterDataInfo->meterOrderIdx, setExecutionContext(pSupporter, pSupporter->pResult, pOneMeterDataInfo->meterOrderIdx,
pOneMeterDataInfo->groupIdx, pMeterQueryInfo); pOneMeterDataInfo->groupIdx, pMeterQueryInfo);
} else { // interval query } else { // interval query
int32_t ret = setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo); ret = setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
tfree(pReqMeterDataInfo); // error code has been set tfree(pReqMeterDataInfo); // error code has been set
pQInfo->killed = 1; pQInfo->killed = 1;
......
...@@ -13,10 +13,7 @@ ...@@ -13,10 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <assert.h> #include "os.h"
#include <stdio.h>
#include <stdlib.h>
#include "taosmsg.h" #include "taosmsg.h"
#include "tlog.h" #include "tlog.h"
#include "tlosertree.h" #include "tlosertree.h"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册