From e8d100657bcf800fd51f8e3a843e41b0c54ff6cd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 3 Aug 2021 18:37:57 +0800 Subject: [PATCH] [TD-5694]: fix memory alloc --- src/common/inc/tdataformat.h | 4 +-- src/common/src/tdataformat.c | 61 +++++++++++++++++++++++------------- src/tsdb/src/tsdbReadImpl.c | 9 +----- 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 99c612c86c..53e77430d3 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -325,6 +325,8 @@ typedef struct SDataCol { #define isAllRowsNull(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } +void tdAllocMemForCol(SDataCol *pCol, int maxPoints); + void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); @@ -358,12 +360,10 @@ typedef struct { int maxRowSize; int maxCols; // max number of columns int maxPoints; // max number of points - //int bufSize; int numOfRows; int numOfCols; // Total number of cols int sversion; // TODO: set sversion - //void * buf; SDataCol *cols; } SDataCols; diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index ad928211a1..077081bfb6 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -22,6 +22,24 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows, bool forceSetNull); +void tdAllocMemForCol(SDataCol *pCol, int maxPoints) { + if(pCol->pData == NULL) { + pCol->pData = malloc(maxPoints * pCol->bytes); + pCol->spaceSize = maxPoints * pCol->bytes; + if(pCol->pData == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize, + strerror(errno)); + } + if(IS_VAR_DATA_TYPE(pCol->type)) { + pCol->dataOff = malloc(maxPoints * sizeof(VarDataOffsetT)); + if(pCol->dataOff == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)(maxPoints * sizeof(VarDataOffsetT)), + strerror(errno)); + } + } + } +} + /** * Duplicate the schema and return a new object */ @@ -214,9 +232,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; - pDataCol->spaceSize = pDataCol->bytes * maxPoints; - pDataCol->pData = NULL; - pDataCol->dataOff = NULL; } // value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { @@ -232,14 +247,7 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP // Find the first not null value, fill all previouse values as NULL dataColSetNEleNull(pCol, numOfRows, maxPoints); } else { - if(pCol->pData == NULL) { - pCol->pData = malloc(maxPoints * pCol->bytes); - ASSERT(pCol->pData != NULL); - if(IS_VAR_DATA_TYPE(pCol->type)) { - pCol->dataOff = malloc(maxPoints * sizeof(VarDataOffsetT)); - ASSERT(pCol->dataOff != NULL); - } - } + tdAllocMemForCol(pCol, maxPoints); } } @@ -277,15 +285,8 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { - - if(pCol->pData == NULL) { - pCol->pData = malloc(maxPoints * pCol->bytes); - ASSERT(pCol->pData != NULL); - if(IS_VAR_DATA_TYPE(pCol->type)) { - pCol->dataOff = malloc(maxPoints * sizeof(VarDataOffsetT)); - ASSERT(pCol->dataOff != NULL); - } - } + if(isAllRowsNull(pCol)) return; + tdAllocMemForCol(pCol, maxPoints); if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; @@ -340,9 +341,24 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { } int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + int i; + int oldMaxCols = pCols->maxCols; + if(oldMaxCols > 0) { + for(i = 0; i < oldMaxCols; i++) { + if(i >= pSchema->numOfCols || + (pCols->cols[i].spaceSize < pSchema->columns[i].bytes * pCols->maxPoints)) { + tfree(pCols->cols[i].pData); + tfree(pCols->cols[i].dataOff); + } + } + } if (schemaNCols(pSchema) > pCols->maxCols) { pCols->maxCols = schemaNCols(pSchema); pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + for(i = oldMaxCols; i < pCols->maxCols; i++) { + pCols->cols[i].pData = NULL; + pCols->cols[i].dataOff = NULL; + } if (pCols->cols == NULL) return -1; } @@ -353,7 +369,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { tdResetDataCols(pCols); pCols->numOfCols = schemaNCols(pSchema); - for (int i = 0; i < schemaNCols(pSchema); i++) { + for (i = 0; i < schemaNCols(pSchema); i++) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } @@ -392,7 +408,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].bytes = pDataCols->cols[i].bytes; pRet->cols[i].offset = pDataCols->cols[i].offset; - pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; + pRet->cols[i].spaceSize = 0; pRet->cols[i].len = 0; pRet->cols[i].dataOff = NULL; pRet->cols[i].pData = NULL; @@ -400,6 +416,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { pRet->cols[i].len = pDataCols->cols[i].len; if (pDataCols->cols[i].len > 0) { + pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; pRet->cols[i].pData = malloc(pDataCols->cols[i].bytes * pDataCols->maxPoints); memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index a16c3ffe6a..711c32535b 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -518,14 +518,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 return -1; } - if(pDataCol->pData == NULL) { - pDataCol->pData = malloc(maxPoints * pDataCol->bytes); - ASSERT(pDataCol->pData != NULL); - if(IS_VAR_DATA_TYPE(pDataCol->type)) { - pDataCol->dataOff = malloc(maxPoints * sizeof(VarDataOffsetT)); - ASSERT(pDataCol->dataOff != NULL); - } - } + tdAllocMemForCol(pDataCol, maxPoints); // Decode the data if (comp) { -- GitLab