refactor some codes, remove the mmap of header file for super table query. issue #969 [tbase-1368]

src/os/linux/inc/os.h

@@ -79,7 +79,7 @@ extern "C" {
   {                        \
     if (FD_VALID(x)) {     \
       close(x);            \
-      x = -1;              \
+      x = FD_INITIALIZER;  \
     }                      \
   }
   

src/system/detail/inc/vnodeQueryImpl.h

@@ -180,8 +180,8 @@ void queryOnBlock(SMeterQuerySupportObj* pSupporter, int64_t* primaryKeys, int32
                   SBlockInfo* pBlockBasicInfo, SMeterDataInfo* pDataHeadInfoEx, SField* pFields,
                   __block_search_fn_t searchFn);
 
-SMeterDataInfo** vnodeFilterQualifiedMeters(SQInfo* pQInfo, int32_t vid, int32_t fileIndex,
-                                            tSidSet* pSidSet, SMeterDataInfo* pMeterDataInfo, int32_t* numOfMeters);
+int32_t vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, tSidSet *pSidSet, SMeterDataInfo *pMeterDataInfo,
+                                   int32_t *numOfMeters, SMeterDataInfo ***pReqMeterDataInfo);
 int32_t vnodeGetVnodeHeaderFileIdx(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
 
 int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters,
@@ -196,11 +196,11 @@ int32_t setIntervalQueryExecutionContext(SMeterQuerySupportObj* pSupporter, int3
 int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange);
 int64_t getNextAccessedKeyInData(SQuery* pQuery, int64_t* pPrimaryCol, SBlockInfo* pBlockInfo, int32_t blockStatus);
 
-uint32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, char* pHeaderData,
-                                int32_t numOfMeters, const char* filePath, SMeterDataInfo** pMeterDataInfo);
+int32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters,
+                                const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks);
 int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv,
                               int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand);
-char *vnodeGetHeaderFileData(SQueryRuntimeEnv *pRuntimeEnv, int32_t vnodeId, int32_t fileIndex);
+int32_t vnodeGetHeaderFile(SQueryRuntimeEnv *pRuntimeEnv, int32_t fileIndex);
 
 /**
  * Create SMeterQueryInfo.

src/system/detail/inc/vnodeRead.h

@@ -100,12 +100,11 @@ typedef struct SQueryFilesInfo {
   uint32_t         numOfFiles;  // the total available number of files for this virtual node during query execution
   int32_t          current;     // the memory mapped header file, NOTE: only one header file can be mmap.
   int32_t          vnodeId;
-
-  int32_t headerFd;         // header file fd
-  char*   pHeaderFileData;  // mmap header files
-  int64_t headFileSize;
-  int32_t dataFd;
-  int32_t lastFd;
+  
+  int32_t          headerFd;         // header file fd
+  int64_t          headerFileSize;
+  int32_t          dataFd;
+  int32_t          lastFd;
 
   char headerFilePath[PATH_MAX];  // current opened header file name
   char dataFilePath[PATH_MAX];    // current opened data file name
@@ -165,11 +164,10 @@ typedef struct SMeterDataInfo {
   uint64_t     offsetInHeaderFile;
   int32_t      numOfBlocks;
   int32_t      start;  // start block index
-  SCompBlock** pBlock;
+  SCompBlock*  pBlock;
   int32_t      meterOrderIdx;
   SMeterObj*   pMeterObj;
-  int32_t      groupIdx;  // group id in meter list
-
+  int32_t      groupIdx;    // group id in meter list
   SMeterQueryInfo* pMeterQInfo;
 } SMeterDataInfo;
 

src/system/detail/src/vnodeQueryProcess.c

@@ -295,8 +295,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
     pQuery->fileId = fid;
     pSummary->numOfFiles++;
 
-    char *pHeaderFileData = vnodeGetHeaderFileData(pRuntimeEnv, vnodeId, fileIdx);
-    if (pHeaderFileData == NULL) { // failed to mmap header file into buffer, ignore current file, try next
+    if (vnodeGetHeaderFile(pRuntimeEnv, fileIdx) != TSDB_CODE_SUCCESS) {
       fid += step;
       continue;
     }
@@ -304,14 +303,16 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
     int32_t numOfQualifiedMeters = 0;
     assert(fileIdx == pRuntimeEnv->vnodeFileInfo.current);
     
-    SMeterDataInfo **pReqMeterDataInfo = vnodeFilterQualifiedMeters(pQInfo, vnodeId, fileIdx, pSupporter->pSidSet,
-        pMeterDataInfo, &numOfQualifiedMeters);
-
-    if (pReqMeterDataInfo == NULL) {
-      dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo);
-
-      pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY;
+    SMeterDataInfo **pReqMeterDataInfo = NULL;
+    int32_t ret = vnodeFilterQualifiedMeters(pQInfo, vnodeId, pSupporter->pSidSet, pMeterDataInfo,
+                                             &numOfQualifiedMeters, &pReqMeterDataInfo);
+    if (ret != TSDB_CODE_SUCCESS) {
+      dError("QInfo:%p failed to create meterdata struct to perform query processing, abort", pQInfo);
+  
+      tfree(pReqMeterDataInfo);
+      pQInfo->code = -ret;
       pQInfo->killed = 1;
+      
       return NULL;
     }
 
@@ -324,8 +325,18 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
       continue;
     }
 
-    uint32_t numOfBlocks = getDataBlocksForMeters(pSupporter, pQuery, pHeaderFileData, numOfQualifiedMeters,
-        pVnodeFileInfo->headerFilePath, pReqMeterDataInfo);
+    uint32_t numOfBlocks = 0;
+    ret = getDataBlocksForMeters(pSupporter, pQuery, numOfQualifiedMeters, pVnodeFileInfo->headerFilePath,
+                                 pReqMeterDataInfo, &numOfBlocks);
+    if (ret != TSDB_CODE_SUCCESS) {
+      dError("QInfo:%p failed to get data block before scan data blocks, abort", pQInfo);
+  
+      tfree(pReqMeterDataInfo);
+      pQInfo->code = -ret;
+      pQInfo->killed = 1;
+  
+      return NULL;
+    }
 
     dTrace("QInfo:%p file:%s, %d meters contains %d blocks to be checked", pQInfo, pVnodeFileInfo->dataFilePath,
            numOfQualifiedMeters, numOfBlocks);
@@ -336,13 +347,13 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
       continue;
     }
 
-    int32_t n = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks,
+    ret = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks,
                                        &nAllocBlocksInfoSize, (int64_t)pQInfo);
-    if (n < 0) {  // failed to create data blocks
-      dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo);
+    if (ret != TSDB_CODE_SUCCESS) {  // failed to create data blocks
+      dError("QInfo:%p build blockInfoEx failed, abort", pQInfo);
       tfree(pReqMeterDataInfo);
 
-      pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY;
+      pQInfo->code = -ret;
       pQInfo->killed = 1;
       return NULL;
     }
@@ -397,7 +408,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMe
         setExecutionContext(pSupporter, pSupporter->pResult, pOneMeterDataInfo->meterOrderIdx,
                             pOneMeterDataInfo->groupIdx, pMeterQueryInfo);
       } else {  // interval query
-        int32_t ret = setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo);
+        ret = setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo);
         if (ret != TSDB_CODE_SUCCESS) {
           tfree(pReqMeterDataInfo);  // error code has been set
           pQInfo->killed = 1;

src/util/src/tlosertree.c

@@ -13,10 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-
+#include "os.h"
 #include "taosmsg.h"
 #include "tlog.h"
 #include "tlosertree.h"