Merge branch 'enh/addTtlToStream' into enh/rocksdbSstateMerge

include/libs/stream/streamState.h

@@ -27,21 +27,10 @@ extern "C" {
 #ifndef _STREAM_STATE_H_
 #define _STREAM_STATE_H_
 
-typedef struct {
-  rocksdb_t*              db;
-  rocksdb_writeoptions_t* writeOpts;
-  rocksdb_readoptions_t*  readOpts;
-  rocksdb_options_t*      dbOpt;
-  void*                   param;
-  void*                   env;
-  rocksdb_cache_t*        cache;
-  TdThreadMutex           mutex;
-  SList*                  list;
-} SBackendHandle;
-void*      streamBackendInit(const char* path);
-void       streamBackendCleanup(void* arg);
-SListNode* streamBackendAddCompare(void* backend, void* arg);
-void       streamBackendDelCompare(void* backend, void* arg);
+// void*      streamBackendInit(const char* path);
+// void       streamBackendCleanup(void* arg);
+// SListNode* streamBackendAddCompare(void* backend, void* arg);
+// void       streamBackendDelCompare(void* backend, void* arg);
 typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
 
 typedef struct STdbState {
@@ -55,8 +44,9 @@ typedef struct STdbState {
   void*                            param;
   void*                            env;
   SListNode*                       pComparNode;
-  SBackendHandle*                  pBackendHandle;
+  void*                            pBackendHandle;
   char                             idstr[48];
+  void*                            compactFactory;
 
   TDB* db;
   TTB* pStateDb;
@@ -168,11 +158,11 @@ typedef struct SStateSessionKey {
   int64_t     opNum;
 } SStateSessionKey;
 
-typedef struct streamValue {
+typedef struct SStreamValue {
   int64_t unixTimestamp;
   int32_t len;
-  char    data[0];
-} streamValue;
+  char*   data;
+} SStreamValue;
 
 int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2);
 int sessionWinKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2);

source/libs/stream/inc/streamBackendRocksdb.h

@@ -17,7 +17,9 @@
 #define _STREAM_BACKEDN_ROCKSDB_H_
 
 #include "executor.h"
-#include "streamInc.h"
+
+#include "rocksdb/c.h"
+// #include "streamInc.h"
 #include "streamState.h"
 #include "tcoding.h"
 #include "tcommon.h"
@@ -28,58 +30,84 @@ typedef struct SCfComparator {
   rocksdb_comparator_t** comp;
   int32_t                numOfComp;
 } SCfComparator;
+
+typedef struct {
+  rocksdb_t*                         db;
+  rocksdb_writeoptions_t*            writeOpts;
+  rocksdb_readoptions_t*             readOpts;
+  rocksdb_options_t*                 dbOpt;
+  void*                              param;
+  void*                              env;
+  rocksdb_cache_t*                   cache;
+  TdThreadMutex                      mutex;
+  rocksdb_compactionfilterfactory_t* filterFactory;
+  SList*                             list;
+} SBackendHandle;
+
+void*      streamBackendInit(const char* path);
+void       streamBackendCleanup(void* arg);
+SListNode* streamBackendAddCompare(void* backend, void* arg);
+void       streamBackendDelCompare(void* backend, void* arg);
+
 int  streamStateOpenBackend(void* backend, SStreamState* pState);
 void streamStateCloseBackend(SStreamState* pState, bool remove);
 void streamStateDestroyCompar(void* arg);
-// void streamStateRemoveBackend(SStreamState* pState);
 
-int32_t streamStateFuncPut_rocksdb(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen);
-int32_t streamStateFuncGet_rocksdb(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen);
-int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key);
+// state cf
 int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
 int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key);
-int32_t streamStateFillPut_rocksdb(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
-int32_t streamStateFillGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
-int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key);
 int32_t streamStateClear_rocksdb(SStreamState* pState);
+int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur);
+int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key);
+int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+int32_t streamStateCurPrev_rocksdb(SStreamState* pState, SStreamStateCur* pCur);
+int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key);
+SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key);
+SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinKey* key);
+SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key);
+
+// func cf
+int32_t streamStateFuncPut_rocksdb(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen);
+int32_t streamStateFuncGet_rocksdb(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen);
+int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key);
 
+//  session cf
 int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen);
+int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
+int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* key);
 SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key);
 SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key);
 SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key);
 int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
-int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur);
 int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey);
-int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
-int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* key);
 int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
                                                 int32_t* pVLen);
+int32_t streamStateSessionClear_rocksdb(SStreamState* pState);
 
 int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData,
                                               int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
 
-int32_t          streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key);
-int32_t          streamStateSessionClear_rocksdb(SStreamState* pState);
-int32_t          streamStateCurPrev_rocksdb(SStreamState* pState, SStreamStateCur* pCur);
-SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinKey* key);
-
-int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
-
-int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
-SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key);
+// fill cf
+int32_t          streamStateFillPut_rocksdb(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+int32_t          streamStateFillGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+int32_t          streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key);
 SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key);
-SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key);
-int32_t          streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
 int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
-SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key);
 SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key);
 SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key);
-int32_t          streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen);
-int32_t          streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen);
+
+// partag cf
+int32_t streamStatePutParTag_rocksdb(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen);
+int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen);
+
+// parname cf
 int32_t streamStatePutParName_rocksdb(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]);
 int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, void** pVal);
-void    streamStateDestroy_rocksdb(SStreamState* pState, bool remove);
+
+void streamStateDestroy_rocksdb(SStreamState* pState, bool remove);
 
 void*   streamStateCreateBatch();
 int32_t streamStateGetBatchSize(void* pBatch);
@@ -89,10 +117,10 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr
                             void* val, int32_t vlen);
 int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch);
 
+// default cf
 int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen);
 int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen);
 int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key);
-
 int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result);
 void*   streamDefaultIterCreate_rocksdb(SStreamState* pState);
 int32_t streamDefaultIterValid_rocksdb(void* iter);
@@ -101,5 +129,13 @@ void    streamDefaultIterNext_rocksdb(void* iter);
 char*   streamDefaultIterKey_rocksdb(void* iter, int32_t* len);
 char*   streamDefaultIterVal_rocksdb(void* iter, int32_t* len);
 
+// batch func
+void*   streamStateCreateBatch();
+int32_t streamStateGetBatchSize(void* pBatch);
+void    streamStateClearBatch(void* pBatch);
+void    streamStateDestroyBatch(void* pBatch);
+int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key,
+                            void* val, int32_t vlen);
+int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch);
 // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result);
 #endif
\ No newline at end of file

source/libs/stream/src/streamMeta.c

@@ -14,6 +14,7 @@
  */
 
 #include "executor.h"
+#include "streamBackendRocksdb.h"
 #include "streamInc.h"
 #include "ttimer.h"
 

source/libs/stream/src/streamState.c

@@ -26,88 +26,6 @@
 
 #define MAX_TABLE_NAME_NUM 100000
 
-void* streamBackendInit(const char* path) {
-  SBackendHandle* pHandle = calloc(1, sizeof(SBackendHandle));
-  pHandle->list = tdListNew(sizeof(SCfComparator));
-  taosThreadMutexInit(&pHandle->mutex, NULL);
-
-  rocksdb_env_t* env = rocksdb_create_default_env();  // rocksdb_envoptions_create();
-  rocksdb_env_set_low_priority_background_threads(env, 4);
-  rocksdb_env_set_high_priority_background_threads(env, 2);
-
-  rocksdb_cache_t* cache = rocksdb_cache_create_lru(128 << 20);
-
-  rocksdb_options_t* opts = rocksdb_options_create();
-  rocksdb_options_set_env(opts, env);
-  rocksdb_options_set_create_if_missing(opts, 1);
-  rocksdb_options_set_create_missing_column_families(opts, 1);
-  rocksdb_options_set_write_buffer_size(opts, 128 << 20);
-  rocksdb_options_set_max_total_wal_size(opts, 128 << 20);
-  rocksdb_options_set_recycle_log_file_num(opts, 6);
-  rocksdb_options_set_max_write_buffer_number(opts, 3);
-
-  pHandle->env = env;
-  pHandle->dbOpt = opts;
-  pHandle->cache = cache;
-
-  char* err = NULL;
-  pHandle->db = rocksdb_open(opts, path, &err);
-  if (err != NULL) {
-    qError("failed to open rocksdb, path:%s, reason:%s", path, err);
-    taosMemoryFreeClear(err);
-    goto _EXIT;
-  }
-
-  return pHandle;
-_EXIT:
-  rocksdb_options_destroy(opts);
-  rocksdb_cache_destroy(cache);
-  rocksdb_env_destroy(env);
-  taosThreadMutexDestroy(&pHandle->mutex);
-  tdListFree(pHandle->list);
-  free(pHandle);
-  return NULL;
-}
-void streamBackendCleanup(void* arg) {
-  SBackendHandle* pHandle = (SBackendHandle*)arg;
-  rocksdb_close(pHandle->db);
-  rocksdb_options_destroy(pHandle->dbOpt);
-  rocksdb_env_destroy(pHandle->env);
-  rocksdb_cache_destroy(pHandle->cache);
-
-  taosThreadMutexDestroy(&pHandle->mutex);
-  SListNode* head = tdListPopHead(pHandle->list);
-  while (head != NULL) {
-    streamStateDestroyCompar(head->data);
-    taosMemoryFree(head);
-    head = tdListPopHead(pHandle->list);
-  }
-  tdListFree(pHandle->list);
-
-  taosMemoryFree(pHandle);
-
-  return;
-}
-SListNode* streamBackendAddCompare(void* backend, void* arg) {
-  SBackendHandle* pHandle = (SBackendHandle*)backend;
-  SListNode*      node = NULL;
-  taosThreadMutexLock(&pHandle->mutex);
-  node = tdListAdd(pHandle->list, arg);
-  taosThreadMutexUnlock(&pHandle->mutex);
-  return node;
-}
-void streamBackendDelCompare(void* backend, void* arg) {
-  SBackendHandle* pHandle = (SBackendHandle*)backend;
-  SListNode*      node = NULL;
-  taosThreadMutexLock(&pHandle->mutex);
-  node = tdListPopNode(pHandle->list, arg);
-  taosThreadMutexUnlock(&pHandle->mutex);
-  if (node) {
-    streamStateDestroyCompar(node->data);
-    taosMemoryFree(node);
-  }
-}
-
 int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) {
   if (pWin1->groupId > pWin2->groupId) {
     return 1;