From a0621dab99723b4c3361e67869963adbccf9786a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 9 Oct 2019 22:27:58 +0800 Subject: [PATCH] update the skiplist node level generate method, and signature of some functions. --- src/inc/tskiplist.h | 30 ++------ src/inc/tsql.h | 2 +- src/system/src/mgmtDb.c | 2 +- src/system/src/mgmtMeter.c | 6 +- src/system/src/vnodeStore.c | 2 - src/util/src/thistogram.c | 13 ++-- src/util/src/tskiplist.c | 138 ++++++++++-------------------------- 7 files changed, 55 insertions(+), 138 deletions(-) diff --git a/src/inc/tskiplist.h b/src/inc/tskiplist.h index 97a582f719..e20dfc613a 100644 --- a/src/inc/tskiplist.h +++ b/src/inc/tskiplist.h @@ -29,15 +29,6 @@ extern "C" { #include "os.h" #include "ttypes.h" -/* - * generate random data with uniform&skewed distribution, extracted from levelDB - */ -typedef struct SRandom { - uint32_t s; - - uint32_t (*rand)(struct SRandom *, int32_t n); -} SRandom; - /* * key of each node * todo move to as the global structure in all search codes... @@ -54,10 +45,11 @@ typedef enum tSkipListPointQueryType { typedef struct tSkipListNode { uint16_t nLevel; char * pData; - tSkipListKey key; struct tSkipListNode **pForward; struct tSkipListNode **pBackward; + + tSkipListKey key; } tSkipListNode; /* @@ -133,9 +125,6 @@ typedef struct tSkipList { __compar_fn_t comparator; pthread_rwlock_t lock; // will be removed soon - // random generator - SRandom r; - // skiplist state tSkipListState state; } tSkipList; @@ -150,14 +139,14 @@ typedef struct tSKipListQueryCond { tSkipListKey upperBnd; int32_t lowerBndRelOptr; // relation operator to denote if lower bound is + // included or not int32_t upperBndRelOptr; } tSKipListQueryCond; -int32_t tSkipListCreate(tSkipList **pSkipList, int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen, - int32_t (*funcp)()); +tSkipList* tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen); -void tSkipListDestroy(tSkipList **pSkipList); +void* tSkipListDestroy(tSkipList *pSkipList); // create skip list key tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength); @@ -170,8 +159,7 @@ tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKe /* * get only *one* node of which key is equalled to pKey, even there are more - * than - * one nodes are of the same key + * than one nodes are of the same key */ tSkipListNode *tSkipListGetOne(tSkipList *pSkipList, tSkipListKey *pKey); @@ -215,12 +203,6 @@ int32_t tSkipListPointQuery(tSkipList *pSkipList, tSkipListKey *pKey, int32_t nu tSkipListNode ***pResult); void removeNodeEachLevel(tSkipList *pSkipList, int32_t nLevel); - -// todo move to utility -void tInitMatrix(double *x, double *y, int32_t length, double p[2][3]); - -int32_t tCompute(double p[2][3]); - #ifdef __cplusplus } #endif diff --git a/src/inc/tsql.h b/src/inc/tsql.h index 34f507b516..d0cf049a24 100644 --- a/src/inc/tsql.h +++ b/src/inc/tsql.h @@ -48,7 +48,7 @@ enum { TSQL_NODE_TYPE_VALUE = 0x4, }; -extern char tTokenTypeSwitcher[12]; +extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ do { \ diff --git a/src/system/src/mgmtDb.c b/src/system/src/mgmtDb.c index ddd6846037..a2c409be9c 100644 --- a/src/system/src/mgmtDb.c +++ b/src/system/src/mgmtDb.c @@ -478,7 +478,7 @@ int mgmtRemoveMetricFromDb(SDbObj *pDb, STabObj *pMetric) { pDb->numOfMetrics--; if (pMetric->pSkipList != NULL) { - tSkipListDestroy(&pMetric->pSkipList); + pMetric->pSkipList = tSkipListDestroy(pMetric->pSkipList); } return 0; } diff --git a/src/system/src/mgmtMeter.c b/src/system/src/mgmtMeter.c index dc1fa004ea..ffc267f559 100644 --- a/src/system/src/mgmtMeter.c +++ b/src/system/src/mgmtMeter.c @@ -34,7 +34,7 @@ extern int64_t sdbVersion; #define mgmtDestroyMeter(pMeter) \ do { \ tfree(pMeter->schema); \ - tSkipListDestroy(&(pMeter->pSkipList));\ + pMeter->pSkipList = tSkipListDestroy((pMeter)->pSkipList);\ tfree(pMeter); \ } while (0) @@ -781,8 +781,8 @@ static void addMeterIntoMetricIndex(STabObj *pMetric, STabObj *pMeter) { SSchema * pTagSchema = (SSchema *)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema)); if (pMetric->pSkipList == NULL) { - tSkipListCreate(&pMetric->pSkipList, MAX_SKIP_LIST_LEVEL, pTagSchema[KEY_COLUMN_OF_TAGS].type, - pTagSchema[KEY_COLUMN_OF_TAGS].bytes, tSkipListDefaultCompare); + pMetric->pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, pTagSchema[KEY_COLUMN_OF_TAGS].type, + pTagSchema[KEY_COLUMN_OF_TAGS].bytes); } if (pMetric->pSkipList) { diff --git a/src/system/src/vnodeStore.c b/src/system/src/vnodeStore.c index d596f50b61..bd8f251c37 100644 --- a/src/system/src/vnodeStore.c +++ b/src/system/src/vnodeStore.c @@ -26,8 +26,6 @@ #include "vnodeStore.h" #include "vnodeUtil.h" -#pragma GCC diagnostic ignored "-Wformat-overflow=" - int vnodeCreateMeterObjFile(int vnode); int tsMaxVnode = -1; diff --git a/src/util/src/thistogram.c b/src/util/src/thistogram.c index f8294df4dd..2c8dd22d02 100644 --- a/src/util/src/thistogram.c +++ b/src/util/src/thistogram.c @@ -133,7 +133,7 @@ SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); #if !defined(USE_ARRAYLIST) - tSkipListCreate(&pHisto->pList, MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double), NULL); + pHisto->pList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); pss->numOfEntries = pHisto->maxEntries; pss->pSkipList = pHisto->pList; @@ -623,11 +623,14 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); + // error in histogram info + if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { + return pResHistogram; + } + SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); + int32_t i = 0, j = 0, k = 0; - int32_t i = 0; - int32_t j = 0; - int32_t k = 0; while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { if (pHisto1->elems[i].val < pHisto2->elems[j].val) { pHistoBins[k++] = pHisto1->elems[i++]; @@ -660,8 +663,8 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 histogramMergeImpl(pHistoBins, &k); } - memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * numOfEntries); pResHistogram->numOfEntries = k; + memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); free(pHistoBins); return pResHistogram; diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 66f45b71f7..861d87b1f5 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -23,28 +23,6 @@ #include "tskiplist.h" #include "tutil.h" -static uint32_t doGetRand(SRandom *pRand, int32_t n) { - const uint32_t val = 2147483647L; - uint64_t p = pRand->s * 16807; - - pRand->s = (uint32_t)((p >> 31) + (p & val)); - if (pRand->s > val) { - pRand->s -= val; - } - - return (pRand->s % n); -} - -static SRandom getRand(uint32_t s) { - uint32_t seed = s & 0x7FFFFFFF; - if (seed == 0 || seed == INT32_MAX) { - seed = 1; - } - - struct SRandom r = {seed, doGetRand}; - return r; -} - void recordNodeEachLevel(tSkipList *pSkipList, int32_t nLevel); int32_t getSkipListNodeLevel(tSkipList *pSkipList); @@ -55,14 +33,14 @@ static int32_t getSkipListNodeRandomHeight(tSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; - while ((pSkipList->r.rand(&pSkipList->r, MAX_SKIP_LIST_LEVEL) % factor) == 0 && n <= MAX_SKIP_LIST_LEVEL) { + while ((rand() % factor) == 0 && n <= pSkipList->nMaxLevel) { n++; } return n; } -void tSkipListDoRecordPutNode(tSkipList *pSkipList) { +void tSkipListDoRecordPut(tSkipList *pSkipList) { const int32_t MAX_RECORD_NUM = 1000; if (pSkipList->state.nInsertObjs == MAX_RECORD_NUM) { @@ -239,33 +217,33 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { return comparator; } -int32_t tSkipListCreate(tSkipList **pSkipList, int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen, - int32_t (*funcp)()) { - (*pSkipList) = (tSkipList *)calloc(1, sizeof(tSkipList)); - if ((*pSkipList) == NULL) { - return -1; +tSkipList* tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen) { + tSkipList *pSkipList = (tSkipList *)calloc(1, sizeof(tSkipList)); + if (pSkipList == NULL) { + return NULL; } - (*pSkipList)->keyType = keyType; + pSkipList->keyType = keyType; - (*pSkipList)->comparator = getKeyComparator(keyType); - (*pSkipList)->pHead.pForward = (tSkipListNode **)calloc(1, POINTER_BYTES * MAX_SKIP_LIST_LEVEL); + pSkipList->comparator = getKeyComparator(keyType); + pSkipList->pHead.pForward = (tSkipListNode **)calloc(1, POINTER_BYTES * MAX_SKIP_LIST_LEVEL); - (*pSkipList)->nMaxLevel = MAX_SKIP_LIST_LEVEL; - (*pSkipList)->nLevel = 1; + pSkipList->nMaxLevel = MAX_SKIP_LIST_LEVEL; + pSkipList->nLevel = 1; - (*pSkipList)->nMaxKeyLen = nMaxKeyLen; - (*pSkipList)->nMaxLevel = nMaxLevel; + pSkipList->nMaxKeyLen = nMaxKeyLen; + pSkipList->nMaxLevel = nMaxLevel; - if (pthread_rwlock_init(&(*pSkipList)->lock, NULL) != 0) { - return -1; + if (pthread_rwlock_init(&pSkipList->lock, NULL) != 0) { + tfree(pSkipList->pHead.pForward); + tfree(pSkipList); + return NULL; } srand(time(NULL)); - (*pSkipList)->r = getRand(time(NULL)); + pSkipList->state.nTotalMemSize += sizeof(tSkipList); - (*pSkipList)->state.nTotalMemSize += sizeof(tSkipList); - return 0; + return pSkipList; } static void doRemove(tSkipList *pSkipList, tSkipListNode *pNode, tSkipListNode *forward[]) { @@ -324,77 +302,33 @@ static tSkipListNode *tSkipListCreateNode(void *pData, const tSkipListKey *pKey, } tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength) { - tSkipListKey k; - k.nType = (uint8_t)type; - - switch (type) { - case TSDB_DATA_TYPE_INT: { - k.i64Key = *(int32_t *)val; - return k; - } - case TSDB_DATA_TYPE_BIGINT: { - k.i64Key = *(int64_t *)val; - return k; - } - case TSDB_DATA_TYPE_DOUBLE: { - k.dKey = *(double *)val; - return k; - } - case TSDB_DATA_TYPE_FLOAT: { - k.dKey = *(float *)val; - return k; - } - case TSDB_DATA_TYPE_SMALLINT: { - k.i64Key = *(int16_t *)val; - return k; - } - case TSDB_DATA_TYPE_TINYINT: { - k.i64Key = *(int8_t *)val; - return k; - } - case TSDB_DATA_TYPE_BOOL: { - k.i64Key = *(int8_t *)val; - return k; - } - case TSDB_DATA_TYPE_BINARY: { - k.pz = malloc(keyLength + 1); - k.nLen = keyLength; - memcpy(k.pz, val, keyLength); - k.pz[keyLength] = 0; - return k; - } - case TSDB_DATA_TYPE_NCHAR: { - k.pz = malloc(keyLength + TSDB_NCHAR_SIZE); - k.nLen = keyLength / TSDB_NCHAR_SIZE; - - wcsncpy(k.wpz, (wchar_t *)val, k.nLen); - k.wpz[k.nLen] = 0; - - return k; - } - default: - return k; - } + tSkipListKey k = {0}; + tVariantCreateB(&k, val, (uint32_t) keyLength, (uint32_t) type); + return k; } void tSkipListDestroyKey(tSkipListKey *pKey) { tVariantDestroy(pKey); } -void tSkipListDestroy(tSkipList **pSkipList) { - if ((*pSkipList) == NULL) { - return; +void* tSkipListDestroy(tSkipList *pSkipList) { + if (pSkipList == NULL) { + return NULL; } - pthread_rwlock_wrlock(&(*pSkipList)->lock); - tSkipListNode *pNode = (*pSkipList)->pHead.pForward[0]; + pthread_rwlock_wrlock(&pSkipList->lock); + tSkipListNode *pNode = pSkipList->pHead.pForward[0]; while (pNode) { tSkipListNode *pTemp = pNode; pNode = pNode->pForward[0]; tfree(pTemp); } - tfree((*pSkipList)->pHead.pForward); - pthread_rwlock_unlock(&(*pSkipList)->lock); - tfree(*pSkipList); + tfree(pSkipList->pHead.pForward); + pthread_rwlock_unlock(&pSkipList->lock); + + pthread_rwlock_destroy(&pSkipList->lock); + tfree(pSkipList); + + return NULL; } tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKey, int32_t insertIdenticalKey) { @@ -405,7 +339,7 @@ tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKe pthread_rwlock_wrlock(&pSkipList->lock); // record one node is put into skiplist - tSkipListDoRecordPutNode(pSkipList); + tSkipListDoRecordPut(pSkipList); tSkipListNode *px = &pSkipList->pHead; @@ -419,9 +353,9 @@ tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKe forward[i] = px; } + // if the skiplist does not allowed identical key inserted, the new data will be discarded. if ((insertIdenticalKey == 0) && forward[0] != &pSkipList->pHead && (pSkipList->comparator(&forward[0]->key, pKey) == 0)) { - /* ignore identical key*/ pthread_rwlock_unlock(&pSkipList->lock); return forward[0]; } -- GitLab