未验证 提交 dfbb9581 编写于 作者: S Shengliang Guan 提交者: GitHub

Merge pull request #17929 from taosdata/perf/sma_optimize

perf: optimize sma calculation
......@@ -134,6 +134,7 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
// STRUCT ================================
struct STColumn {
......
......@@ -923,7 +923,7 @@ char *tTagValToData(const STagVal *value, bool isJson) {
}
bool tTagGet(const STag *pTag, STagVal *pTagVal) {
if(!pTag || !pTagVal){
if (!pTag || !pTagVal) {
return false;
}
......@@ -1723,3 +1723,385 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
_exit:
return code;
}
#define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \
do { \
(SUM) += (VAL); \
if ((MAX) < (VAL)) (MAX) = (VAL); \
if ((MIN) > (VAL)) (MIN) = (VAL); \
} while (0)
static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*sum = 0;
*max = 0;
*min = 1;
*numOfNull = 0;
int8_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0;
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0;
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*sum = 0;
*max = INT8_MIN;
*min = INT8_MAX;
*numOfNull = 0;
int8_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((int8_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((int8_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*sum = 0;
*max = INT16_MIN;
*min = INT16_MAX;
*numOfNull = 0;
int16_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((int16_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((int16_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*sum = 0;
*max = INT32_MIN;
*min = INT32_MAX;
*numOfNull = 0;
int32_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((int32_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((int32_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*sum = 0;
*max = INT64_MIN;
*min = INT64_MAX;
*numOfNull = 0;
int64_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((int64_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((int64_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*sum, *max, *min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(double *)sum = 0;
*(double *)max = -FLT_MAX;
*(double *)min = FLT_MAX;
*numOfNull = 0;
float val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((float *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((float *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(double *)sum = 0;
*(double *)max = -DBL_MAX;
*(double *)min = DBL_MAX;
*numOfNull = 0;
double val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((double *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((double *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT8_MAX;
*numOfNull = 0;
uint8_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((uint8_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((uint8_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT16_MAX;
*numOfNull = 0;
uint16_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((uint16_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((uint16_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT32_MAX;
*numOfNull = 0;
uint32_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((uint32_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((uint32_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT64_MAX;
*numOfNull = 0;
uint64_t val;
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
val = ((uint64_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
(*numOfNull)++;
break;
case 2:
val = ((uint64_t *)pColData->pData)[iVal];
CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val);
break;
default:
ASSERT(0);
break;
}
}
}
}
void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = {
NULL,
tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL
tColDataCalcSMATinyInt, // TSDB_DATA_TYPE_TINYINT
tColDataCalcSMATinySmallInt, // TSDB_DATA_TYPE_SMALLINT
tColDataCalcSMAInt, // TSDB_DATA_TYPE_INT
tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT
tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT
tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE
NULL, // TSDB_DATA_TYPE_VARCHAR
tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP
NULL, // TSDB_DATA_TYPE_NCHAR
tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT
tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT
tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT
tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT
NULL, // TSDB_DATA_TYPE_JSON
NULL, // TSDB_DATA_TYPE_VARBINARY
NULL, // TSDB_DATA_TYPE_DECIMAL
NULL, // TSDB_DATA_TYPE_BLOB
NULL // TSDB_DATA_TYPE_MEDIUMBLOB
};
......@@ -191,7 +191,6 @@ int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now);
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
......
......@@ -427,13 +427,13 @@ static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *p
{tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL
{tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE
};
extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
// extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (pBuilder->calcSma) {
if (COL_VAL_IS_VALUE(pColVal)) {
tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
// tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
} else {
pBuilder->sma.numOfNull++;
}
......
......@@ -519,8 +519,8 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
SColumnDataAgg sma;
tsdbCalcColDataSMA(pColData, &sma);
SColumnDataAgg sma = {.colId = pColData->cid};
tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull);
code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma));
if (code) goto _err;
......
......@@ -1437,111 +1437,6 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
return n;
}
#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \
do { \
(SUM_V) += (VAL); \
if (!(MINSET)) { \
(MIN_V) = (VAL); \
(MINSET) = 1; \
} else if ((MIN_V) > (VAL)) { \
(MIN_V) = (VAL); \
} \
if (!(MAXSET)) { \
(MAX_V) = (VAL); \
(MAXSET) = 1; \
} else if ((MAX_V) < (VAL)) { \
(MAX_V) = (VAL); \
} \
} while (0)
static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int16_t val = *(int16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int32_t val = *(int32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int64_t val = *(int64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
float val = *(float *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
double val = *(double *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint8_t val = *(uint8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint16_t val = *(uint16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
uint32_t val = *(uint32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint64_t val = *(uint64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = {
NULL,
tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL
tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT
tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT
tSmaUpdateInt, // TSDB_DATA_TYPE_INT
tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT
tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT
tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE
NULL, // TSDB_DATA_TYPE_VARCHAR
tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP
NULL, // TSDB_DATA_TYPE_NCHAR
tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT
tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT
tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT
tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT
NULL, // TSDB_DATA_TYPE_JSON
NULL, // TSDB_DATA_TYPE_VARBINARY
NULL, // TSDB_DATA_TYPE_DECIMAL
NULL, // TSDB_DATA_TYPE_BLOB
NULL, // TSDB_DATA_TYPE_MEDIUMBLOB
};
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
*pColAgg = (SColumnDataAgg){.colId = pColData->cid};
uint8_t minSet = 0;
uint8_t maxSet = 0;
SColVal cv;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
tColDataGetValue(pColData, iVal, &cv);
if (COL_VAL_IS_VALUE(&cv)) {
tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet);
} else {
pColAgg->numOfNull++;
}
}
}
int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
int32_t *szOut, uint8_t **ppBuf) {
int32_t code = 0;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册