more

src/util/inc/tarray.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TAOSARRAY_H
+#define TDENGINE_TAOSARRAY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#define TARRAY_MIN_SIZE 8
+#define TARRAY_GET_ELEM(array, index) ((array)->pData + (index) * (array)->elemSize)
+
+typedef struct SArray {
+  size_t size;
+  size_t capacity;
+  size_t elemSize;
+
+  void* pData;
+} SArray;
+
+/**
+ *
+ * @param size
+ * @param elemSize
+ * @return
+ */
+void* taosArrayInit(size_t size, size_t elemSize);
+
+/**
+ *
+ * @param pArray
+ * @param pData
+ * @return
+ */
+void* taosArrayPush(SArray* pArray, void* pData);
+
+/**
+ *
+ * @param pArray
+ */
+void taosArrayPop(SArray* pArray);
+
+/**
+ *
+ * @param pArray
+ * @param index
+ * @return
+ */
+void* taosArrayGet(SArray* pArray, size_t index);
+
+/**
+ *
+ * @param pArray
+ * @return
+ */
+size_t taosArrayGetSize(SArray* pArray);
+
+/**
+ *
+ * @param pArray
+ * @param index
+ * @param pData
+ */
+void taosArrayInsert(SArray* pArray, int32_t index, void* pData);
+
+/**
+ *
+ * @param pArray
+ */
+void taosArrayDestory(SArray* pArray);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TAOSARRAY_H

src/util/inc/tskiplist.h

@@ -20,59 +20,62 @@
 extern "C" {
 #endif
 
-#define MAX_SKIP_LIST_LEVEL 20
-
-#include <pthread.h>
-#include <stdint.h>
-#include <stdlib.h>
-
 #include "os.h"
 #include "ttypes.h"
+#include "tarray.h"
 
 /*
  * key of each node
  * todo move to as the global structure in all search codes...
  */
+#define MAX_SKIP_LIST_LEVEL 15
+#define SKIP_LIST_RECORD_PERFORMANCE 0
+
+typedef char *SSkipListKey;
+typedef char *(*__sl_key_fn_t)(const void *);
+
+/**
+ * the format of skip list node is as follows:
+ * +------------+-----------------------+------------------------+-----+------+
+ * | node level | forward pointer array | backward pointer array | key | data |
+ * +------------+-----------------------+------------------------+-----+------+
+ * the skiplist node is located in a consecutive memory area, key will not be copy to skip list
+ */
+typedef struct SSkipListNode {
+  uint8_t level;
+} SSkipListNode;
 
-const static size_t SKIP_LIST_STR_KEY_LENGTH_THRESHOLD = 15;
-typedef tVariant    tSkipListKey;
-
-typedef enum tSkipListPointQueryType {
-  INCLUDE_POINT_QUERY,
-  EXCLUDE_POINT_QUERY,
-} tSkipListPointQueryType;
+#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
 
-typedef struct tSkipListNode {
-  uint16_t nLevel;
-  char *   pData;
+#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
+#define SL_GET_BACKWARD_POINTER(n, _l) \
+  ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
 
-  struct tSkipListNode **pForward;
-  struct tSkipListNode **pBackward;
+#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level))
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
 
-  tSkipListKey key;
-} tSkipListNode;
+#define SL_GET_NODE_LEVEL(n) *(int32_t *)((n))
 
 /*
- * @version 0.2
+ * @version 0.3
  * @date   2017/11/12
- * the simple version of SkipList.
+ * the simple version of skip list.
+ *
  * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate
- * deterministic result. Later, we will remove the lock in SkipList to further
- * enhance the performance. In this case, one should use the concurrent skip list (by
- * using michael-scott algorithm) instead of this simple version in a multi-thread
- * environment, to achieve higher performance of read/write operations.
+ * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance.
+ * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of
+ * this simple version in a multi-thread environment, to achieve higher performance of read/write operations.
  *
  * Note: Duplicated primary key situation.
  * In case of duplicated primary key, two ways can be employed to handle this situation:
- * 1. add as normal insertion with out special process.
- * 2. add an overflow pointer at each list node, all nodes with the same key will be added
- *    in the overflow pointer. In this case, the total steps of each search will be reduced significantly.
+ * 1. add as normal insertion without special process.
+ * 2. add an overflow pointer at each list node, all nodes with the same key will be added in the overflow pointer.
+ *    In this case, the total steps of each search will be reduced significantly.
  *    Currently, we implement the skip list in a line with the first means, maybe refactor it soon.
  *
  *    Memory consumption: the memory alignment causes many memory wasted. So, employ a memory
  *    pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs.
  *
- * 3. use the iterator pattern to refactor all routines to make it more clean
  */
 
 // state struct, record following information:
@@ -81,7 +84,7 @@ typedef struct tSkipListNode {
 // avg search rsp time, for latest 1000 queries
 // total memory size
 typedef struct tSkipListState {
-  // in bytes, sizeof(tSkipList)+sizeof(tSkipListNode)*tSkipList->nSize
+  // in bytes, sizeof(SSkipList)+sizeof(SSkipListNode)*SSkipList->nSize
   uint64_t nTotalMemSize;
   uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL];
   uint64_t queryCount;  // total query count
@@ -101,68 +104,95 @@ typedef struct tSkipListState {
   uint64_t nTotalElapsedTimeForInsert;
 } tSkipListState;
 
-typedef struct tSkipList {
-  tSkipListNode pHead;
-  uint64_t      nSize;
-  uint16_t      nMaxLevel;
-  uint16_t      nLevel;
-  uint16_t      keyType;
-  uint16_t      nMaxKeyLen;
+typedef struct SSkipListKeyInfo {
+  uint8_t dupKey : 2;  // if allow duplicated key in the skip list
+  uint8_t type : 6;    // key type
+  uint8_t len;         // maximum key length, used in case of string key
+} SSkipListKeyInfo;
+
+typedef struct SSkipList {
+  __compar_fn_t    comparFn;
+  __sl_key_fn_t    keyFn;
+  uint32_t         size;
+  uint8_t          maxLevel;
+  uint8_t          level;
+  SSkipListKeyInfo keyInfo;
 
-  __compar_fn_t    comparator;
-  pthread_rwlock_t lock;   // will be removed soon
+  pthread_rwlock_t *lock;
+  SSkipListNode *   pHead;
+
+#if SKIP_LIST_RECORD_PERFORMANCE
   tSkipListState state;  // skiplist state
-} tSkipList;
+#endif
+
+} SSkipList;
 
 /*
  * iterate the skiplist
  * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
  * continue iterating the skiplist, so add the reference count for skiplist
- * TODO add the ref for skiplist when one iterator is created
+ * TODO add the ref for skip list when one iterator is created
  */
 typedef struct SSkipListIterator {
-  tSkipList *    pSkipList;
-  tSkipListNode *cur;
+  SSkipList *    pSkipList;
+  SSkipListNode *cur;
   int64_t        num;
 } SSkipListIterator;
 
-/*
- * query condition structure to denote the range query
- * todo merge the point query cond with range query condition
+/**
+ *
+ * @param nMaxLevel   maximum skip list level
+ * @param keyType     type of key
+ * @param dupKey      allow the duplicated key in the skip list
+ * @return
  */
-typedef struct tSKipListQueryCond {
-  // when the upper bounding == lower bounding, it is a point query
-  tSkipListKey lowerBnd;
-  tSkipListKey upperBnd;
-  int32_t      lowerBndRelOptr;  // relation operator to denote if lower bound is
-  int32_t      upperBndRelOptr;  // included or not
-} tSKipListQueryCond;
-
-tSkipList *tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen);
+SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
+                           __sl_key_fn_t fn);
 
-void *tSkipListDestroy(tSkipList *pSkipList);
-
-// create skip list key
-tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength);
-
-// destroy skip list key
-void tSkipListDestroyKey(tSkipListKey *pKey);
+/**
+ *
+ * @param pSkipList
+ * @return                NULL will always be returned
+ */
+void *tSkipListDestroy(SSkipList *pSkipList);
 
-// put data into skiplist
-tSkipListNode *tSkipListPut(tSkipList *pSkipList, void *pData, tSkipListKey *pKey, int32_t insertIdenticalKey);
+/**
+ *
+ * @param pSkipList
+ * @param level
+ * @param headSize
+ */
+void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
 
-/*
- * get only *one* node of which key is equalled to pKey, even there are more
- * than one nodes are of the same key
+/**
+ * put the skip list node into the skip list.
+ * If failed, NULL will be returned, otherwise, the pNode will be returned.
+ *
+ * @param pSkipList
+ * @param pNode
+ * @return
  */
-tSkipListNode *tSkipListGetOne(tSkipList *pSkipList, tSkipListKey *pKey);
+SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
 
-/*
- * get all data with the same keys
+/**
+ * get only *one* node of which key is equalled to pKey, even there are more than one nodes are of the same key
+ *
+ * @param pSkipList
+ * @param pKey
+ * @param keyType
+ * @return
  */
-int32_t tSkipListGets(tSkipList *pSkipList, tSkipListKey *pKey, tSkipListNode ***pRes);
+SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType);
 
-int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (*fp)(tSkipListNode *, void *),
+/**
+ *
+ * @param pSkipList
+ * @param pRes
+ * @param fp
+ * @param param
+ * @return
+ */
+int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *),
                              void *param);
 
 /*
@@ -173,30 +203,16 @@ int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (
  * true: one node has been removed
  * false: no node has been removed
  */
-bool tSkipListRemove(tSkipList *pSkipList, tSkipListKey *pKey);
+bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey);
 
 /*
  * remove the specified node in parameters
  */
-void tSkipListRemoveNode(tSkipList *pSkipList, tSkipListNode *pNode);
-
-// for debug purpose only
-void tSkipListPrint(tSkipList *pSkipList, int16_t nlevel);
-
-/*
- * range query & single point query function
- */
-int32_t tSkipListQuery(tSkipList *pSkipList, tSKipListQueryCond *pQueryCond, tSkipListNode ***pResult);
-
-/*
- * include/exclude point query
- */
-int32_t tSkipListPointQuery(tSkipList *pSkipList, tSkipListKey *pKey, int32_t numOfKey, tSkipListPointQueryType type,
-                            tSkipListNode ***pResult);
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
 
-int32_t tSkipListIteratorReset(tSkipList *pSkipList, SSkipListIterator *iter);
+int32_t        tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter);
 bool           tSkipListIteratorNext(SSkipListIterator *iter);
-tSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter);
+SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter);
 
 #ifdef __cplusplus
 }

src/util/src/tarray.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tarray.h"
+
+void* taosArrayInit(size_t size, size_t elemSize) {
+  assert(elemSize > 0);
+
+  if (size < TARRAY_MIN_SIZE) {
+    size = TARRAY_MIN_SIZE;
+  }
+
+  SArray* pArray = calloc(1, sizeof(SArray));
+  if (pArray == NULL) {
+    return NULL;
+  }
+
+  pArray->pData = calloc(size, elemSize * size);
+  if (pArray->pData == NULL) {
+    free(pArray);
+    return NULL;
+  }
+
+  pArray->capacity = size;
+  pArray->elemSize = elemSize;
+  return pArray;
+}
+
+static void taosArrayResize(SArray* pArray) {
+  assert(pArray->size >= pArray->capacity);
+
+  size_t size = pArray->capacity;
+  size = (size << 1u);
+
+  void* tmp = realloc(pArray->pData, size * pArray->elemSize);
+  if (tmp == NULL) {
+    // todo
+  }
+
+  pArray->pData = tmp;
+  pArray->capacity = size;
+}
+
+void* taosArrayPush(SArray* pArray, void* pData) {
+  if (pArray == NULL || pData == NULL) {
+    return NULL;
+  }
+
+  if (pArray->size >= pArray->capacity) {
+    taosArrayResize(pArray);
+  }
+
+  void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
+  memcpy(dst, pData, pArray->elemSize);
+
+  pArray->size += 1;
+  return dst;
+}
+
+void taosArrayPop(SArray* pArray) {
+  if (pArray == NULL || pArray->size == 0) {
+    return;
+  }
+
+  pArray->size -= 1;
+}
+
+void* taosArrayGet(SArray* pArray, size_t index) {
+  assert(index < pArray->size);
+  return TARRAY_GET_ELEM(pArray, index);
+}
+
+size_t taosArrayGetSize(SArray* pArray) { return pArray->size; }
+
+void taosArrayInsert(SArray* pArray, int32_t index, void* pData) {
+  if (pArray == NULL || pData == NULL) {
+    return;
+  }
+
+  if (index >= pArray->size) {
+    taosArrayPush(pArray, pData);
+    return;
+  }
+
+  if (pArray->size >= pArray->capacity) {
+    taosArrayResize(pArray);
+  }
+
+  void* dst = TARRAY_GET_ELEM(pArray, index);
+
+  int32_t remain = pArray->size - index;
+  memmove(dst + pArray->elemSize, dst, pArray->elemSize * remain);
+  memcpy(dst, pData, pArray->elemSize);
+
+  pArray->size += 1;
+}
+
+void taosArrayDestory(SArray* pArray) {
+  if (pArray == NULL) {
+    return;
+  }
+
+  free(pArray->pData);
+  free(pArray);
+}