未验证 提交 c5c611ab 编写于 作者: H haojun Liao 提交者: GitHub

Merge pull request #4358 from taosdata/patch/td-2135

[TD-2135]<patch>: reduce memory usage
...@@ -282,7 +282,7 @@ typedef struct { ...@@ -282,7 +282,7 @@ typedef struct {
int8_t dataSourceType; // load data from file or not int8_t dataSourceType; // load data from file or not
int8_t submitSchema; // submit block is built with table schema int8_t submitSchema; // submit block is built with table schema
STagData tagData; STagData *pTagData; // NOTE: pTagData->data is used as a variant length array
SHashObj *pTableList; // referred table involved in sql SHashObj *pTableList; // referred table involved in sql
SArray *pDataBlocks; // SArray<STableDataBlocks*> submit data blocks after parsing sql SArray *pDataBlocks; // SArray<STableDataBlocks*> submit data blocks after parsing sql
} SSqlCmd; } SSqlCmd;
......
...@@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { ...@@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sql += index; sql += index;
tscAllocPayload(pCmd, sizeof(STagData)); tscAllocPayload(pCmd, sizeof(STagData));
STagData *pTag = &pCmd->tagData;
memset(pTag, 0, sizeof(STagData));
//the source super table is moved to the secondary position of the pTableMetaInfo list //the source super table is moved to the secondary position of the pTableMetaInfo list
if (pQueryInfo->numOfTables < 2) { if (pQueryInfo->numOfTables < 2) {
...@@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { ...@@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return code; return code;
} }
STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data));
if (pTag == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memset(pTag, 0, offsetof(STagData, data));
tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
pCmd->pTagData = pTag;
code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
...@@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { ...@@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
tdSortKVRowByColIdx(row); tdSortKVRowByColIdx(row);
pTag->dataLen = kvRowLen(row);
pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row));
if (pTag == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pCmd->pTagData = pTag;
pTag->dataLen = htonl(kvRowLen(row));
kvRowCpy(pTag->data, row); kvRowCpy(pTag->data, row);
free(row); free(row);
...@@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { ...@@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z);
} }
pTag->dataLen = htonl(pTag->dataLen);
if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
} }
......
...@@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { ...@@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
size_t len = htonl(pCmd->tagData.dataLen); if (pCmd->autoCreated && pCmd->pTagData != NULL) {
if (pSql->cmd.autoCreated) { int len = htonl(pCmd->pTagData->dataLen);
if (len > 0) { if (len > 0) {
len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen); len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen);
memcpy(pInfoMsg->tags, &pCmd->tagData, len); memcpy(pInfoMsg->tags, pCmd->pTagData, len);
pMsg += len; pMsg += len;
} }
} }
...@@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf ...@@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
pNew->signature = pNew; pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_META; pNew->cmd.command = TSDB_SQL_META;
registerSqlObj(pNew);
tscAddSubqueryInfo(&pNew->cmd); tscAddSubqueryInfo(&pNew->cmd);
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
...@@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf ...@@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
tscError("%p malloc failed for payload to get table meta", pSql); tscError("%p malloc failed for payload to get table meta", pSql);
free(pNew); tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
...@@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf ...@@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData));
if (pSql->cmd.pTagData != NULL) {
int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
pNew->cmd.pTagData = calloc(1, size);
if (pNew->cmd.pTagData == NULL) {
tscError("%p malloc failed for new tag data to get table meta", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
}
tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
pNew->fp = tscTableMetaCallBack; pNew->fp = tscTableMetaCallBack;
pNew->param = pSql; pNew->param = pSql;
registerSqlObj(pNew);
int32_t code = tscProcessSql(pNew); int32_t code = tscProcessSql(pNew);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated
......
...@@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { ...@@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
pCmd->pTableList = NULL; pCmd->pTableList = NULL;
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
tscFreeQueryInfo(pCmd, removeFromCache); tscFreeQueryInfo(pCmd, removeFromCache);
} }
...@@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { ...@@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tscFreeSqlResult(pSql); tscFreeSqlResult(pSql);
tscResetSqlCmdObj(pCmd, false); tscResetSqlCmdObj(pCmd, false);
tfree(pCmd->pTagData);
memset(pCmd->payload, 0, (size_t)pCmd->allocSize); memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
tfree(pCmd->payload); tfree(pCmd->payload);
pCmd->allocSize = 0; pCmd->allocSize = 0;
...@@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm ...@@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
pCmd->command = cmd; pCmd->command = cmd;
pCmd->parseFinished = 1; pCmd->parseFinished = 1;
pCmd->autoCreated = pSql->cmd.autoCreated; pCmd->autoCreated = pSql->cmd.autoCreated;
memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData));
if (pSql->cmd.pTagData != NULL) {
int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
pNew->cmd.pTagData = calloc(1, size);
if (pNew->cmd.pTagData == NULL) {
tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0);
free(pNew);
return NULL;
}
memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
}
if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew); tscFreeSqlObj(pNew);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册