support fill history

cmake/rocksdb_CMakeLists.txt.in

 
 # rocksdb
 ExternalProject_Add(rocksdb
-        GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-        GIT_TAG v8.1.1
+	URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
+  	URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b
+  	DOWNLOAD_NO_PROGRESS 1
+	DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" 
 	SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" 
     	CONFIGURE_COMMAND ""
     	BUILD_COMMAND ""
     	INSTALL_COMMAND ""
     	TEST_COMMAND ""
         )
+

include/libs/function/function.h

@@ -129,22 +129,30 @@ typedef struct SSerializeDataHandle {
 } SSerializeDataHandle;
 
 // incremental state storage
-typedef struct STdbState {
+
+typedef struct SBackendWrapper {
   void          *rocksdb;
   void         **pHandle;
   void          *writeOpts;
   void          *readOpts;
   void         **cfOpts;
   void          *dbOpt;
-  struct SStreamTask *pOwner;
   void          *param;
   void          *env;
   SListNode     *pComparNode;
   void          *pBackend;
-  char                idstr[64];
   void          *compactFactory;
   TdThreadRwlock rwLock;
+  bool           remove;
+  int64_t        backendId;
+  char           idstr[64];
+} SBackendWrapper;
+typedef struct STdbState {
+  SBackendWrapper *pBackendWrapper;
+  int64_t          backendWrapperId;
+  char             idstr[64];
 
+  struct SStreamTask *pOwner;
   void               *db;
   void               *pStateDb;
   void               *pFuncStateDb;

include/libs/stream/tstream.h

@@ -366,6 +366,7 @@ typedef struct SStreamMeta {
   void*         streamBackend;
   int64_t       streamBackendRid;
   SHashObj*     pTaskBackendUnique;
+  TdThreadMutex backendMutex;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -584,10 +585,10 @@ int32_t streamSetStatusNormal(SStreamTask* pTask);
 const char* streamGetTaskStatusStr(int32_t status);
 
 // source level
-int32_t streamSetParamForStreamScanner(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow);
+int32_t streamSetParamForStreamScanner(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow);
 int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
 int32_t streamSourceScanHistoryData(SStreamTask* pTask);
-//int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver);
+// int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver);
 int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask);
 
 int32_t streamDispatchTransferStateMsg(SStreamTask* pTask);

source/dnode/mnode/impl/src/mndScheduler.c

@@ -25,7 +25,8 @@
 #define SINK_NODE_LEVEL (0)
 extern bool tsDeployOnSnode;
 
-static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup, int32_t fillHistory);
+static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId,
+                                      SVgObj* pVgroup, int32_t fillHistory);
 static void    setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask);
 
 int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
@@ -101,13 +102,13 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
   return 0;
 }
 
-int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList, SStreamTask* pTask) {
+int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList,
+                                        SStreamTask* pTask) {
   bool isShuffle = false;
 
   if (pStream->fixedSinkVgId == 0) {
     SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb);
     if (pDb != NULL && pDb->cfg.numOfVgroups > 1) {
-
       isShuffle = true;
       pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH;
       pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
@@ -225,7 +226,8 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SArray* pTaskList, SStrea
   return 0;
 }
 
-int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup, int32_t fillHistory) {
+int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup,
+                               int32_t fillHistory) {
   SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, fillHistory, 0, pTaskList);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -248,7 +250,7 @@ static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTas
 
   // todo set the correct ts, which should be last key of queried table.
   pTask->dataRange.window.skey = INT64_MIN;
-  pTask->dataRange.window.ekey = 1685959190000;//taosGetTimestampMs();
+  pTask->dataRange.window.ekey = 1685959190000;  // taosGetTimestampMs();
 
   mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pTask->dataRange.window.skey,
          pTask->dataRange.window.ekey);
@@ -298,7 +300,7 @@ int32_t setEpToDownstreamTask(SStreamTask* pTask, SStreamTask* pDownstream) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if(pDownstream->pUpstreamEpInfoList == NULL) {
+  if (pDownstream->pUpstreamEpInfoList == NULL) {
     pDownstream->pUpstreamEpInfoList = taosArrayInit(4, POINTER_BYTES);
   }
 
@@ -314,7 +316,7 @@ static SArray* addNewTaskList(SArray* pTasksList) {
 
 // set the history task id
 static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) {
-  for(int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
+  for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
     SStreamTask** pStreamTask = taosArrayGet(pTaskList, i);
     SStreamTask** pHTask = taosArrayGet(pHTaskList, i);
 
@@ -367,8 +369,8 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
 
     // new stream task
     SArray** pSinkTaskList = taosArrayGet(pStream->tasks, SINK_NODE_LEVEL);
-    int32_t code = addSourceStreamTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid,
-                                       0, hasExtraSink);
+    int32_t  code =
+        addSourceStreamTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, 0, hasExtraSink);
     if (code != TSDB_CODE_SUCCESS) {
       sdbRelease(pSdb, pVgroup);
       return -1;
@@ -390,8 +392,8 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t uid, SStreamTask* pDownstreamTask, SMnode* pMnode,
-                               SSubplan* pPlan, SVgObj* pVgroup) {
+static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t uid, SStreamTask* pDownstreamTask,
+                               SMnode* pMnode, SSubplan* pPlan, SVgObj* pVgroup) {
   SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, 0, pTaskList);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -400,9 +402,10 @@ static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t ui
 
   // todo set the correct ts, which should be last key of queried table.
   pTask->dataRange.window.skey = INT64_MIN;
-  pTask->dataRange.window.ekey = 1685959190000;//taosGetTimestampMs();
+  pTask->dataRange.window.ekey = 1685959190000;  // taosGetTimestampMs();
 
-  mDebug("s-task:0x%x set time window:%"PRId64" - %"PRId64, pTask->id.taskId, pTask->dataRange.window.skey, pTask->dataRange.window.ekey);
+  mDebug("s-task:0x%x set time window:%" PRId64 " - %" PRId64, pTask->id.taskId, pTask->dataRange.window.skey,
+         pTask->dataRange.window.ekey);
 
   // all the source tasks dispatch result to a single agg node.
   setFixedDownstreamEpInfo(pTask, pDownstreamTask);
@@ -413,8 +416,8 @@ static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t ui
   return setEpToDownstreamTask(pTask, pDownstreamTask);
 }
 
-static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream, int32_t fillHistory,
-                            SStreamTask** pAggTask) {
+static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream,
+                            int32_t fillHistory, SStreamTask** pAggTask) {
   *pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, fillHistory, pStream->conf.triggerParam, pTaskList);
   if (*pAggTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -472,7 +475,8 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan
     SArray* pHSinkNodeList = taosArrayGetP(pStream->pHTasksList, SINK_NODE_LEVEL);
 
     *pHAggTask = NULL;
-    code = doAddAggTask(pStream->hTaskUid, pHAggTaskList, pHSinkNodeList, pMnode, pStream, pStream->conf.fillHistory, pHAggTask);
+    code = doAddAggTask(pStream->hTaskUid, pHAggTaskList, pHSinkNodeList, pMnode, pStream, pStream->conf.fillHistory,
+                        pHAggTask);
     if (code != TSDB_CODE_SUCCESS) {
       if (pSnode != NULL) {
         sdbRelease(pSdb, pSnode);
@@ -538,7 +542,8 @@ static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPl
     }
 
     if (pStream->conf.fillHistory) {
-      code = doAddSourceTask(pHSourceTaskList, pStream->conf.fillHistory, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup);
+      code = doAddSourceTask(pHSourceTaskList, pStream->conf.fillHistory, pStream->hTaskUid, pHDownstreamTask, pMnode,
+                             plan, pVgroup);
       sdbRelease(pSdb, pVgroup);
 
       if (code != TSDB_CODE_SUCCESS) {
@@ -552,7 +557,8 @@ static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPl
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStream, SArray** pCreatedTaskList, int32_t fillHistory) {
+static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStream, SArray** pCreatedTaskList,
+                            int32_t fillHistory) {
   SArray* pSinkTaskList = addNewTaskList(pTasksList);
   if (pStream->fixedSinkVgId == 0) {
     if (mndAddShuffleSinkTasksToStream(pMnode, pSinkTaskList, pStream, fillHistory) < 0) {
@@ -560,7 +566,8 @@ static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStr
       return -1;
     }
   } else {
-    if (mndAddSinkTaskToStream(pStream, pSinkTaskList, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg, fillHistory) < 0) {
+    if (mndAddSinkTaskToStream(pStream, pSinkTaskList, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg,
+                               fillHistory) < 0) {
       // TODO free
       return -1;
     }
@@ -655,8 +662,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
     }
-  }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL){
-    SNode *pAst = NULL;
+  } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
+    SNode* pAst = NULL;
     if (nodesStringToNode(pTopic->ast, &pAst) != 0) {
       mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
       return -1;
@@ -671,7 +678,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     nodesDestroyNode(pAst);
   }
 
-  if(pPlan){
+  if (pPlan) {
     int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
     if (levelNum != 1) {
       qDestroyQueryPlan(pPlan);

source/libs/stream/inc/streamBackendRocksdb.h

@@ -46,6 +46,7 @@ typedef struct {
 
 void*      streamBackendInit(const char* path);
 void       streamBackendCleanup(void* arg);
+void       streamBackendHandleCleanup(void* arg);
 SListNode* streamBackendAddCompare(void* backend, void* arg);
 void       streamBackendDelCompare(void* backend, void* arg);
 

source/libs/stream/inc/streamInc.h

@@ -55,6 +55,7 @@ int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamRe
 SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem);
 
 extern int32_t streamBackendId;
+extern int32_t streamBackendWrapperId;
 
 #ifdef __cplusplus
 }

source/libs/stream/src/streamMeta.c

@@ -21,10 +21,17 @@
 
 static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT;
 int32_t             streamBackendId = 0;
-static void         streamMetaEnvInit() { streamBackendId = taosOpenRef(20, streamBackendCleanup); }
+int32_t             streamBackendWrapperId = 0;
+static void         streamMetaEnvInit() {
+  streamBackendId = taosOpenRef(64, streamBackendCleanup);
+  streamBackendWrapperId = taosOpenRef(64, streamBackendHandleCleanup);
+}
 
 void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
-void streamMetaCleanup() { taosCloseRef(streamBackendId); }
+void streamMetaCleanup() {
+  taosCloseRef(streamBackendId);
+  taosCloseRef(streamBackendWrapperId);
+}
 
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) {
   int32_t      code = -1;
@@ -90,10 +97,14 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 
   pMeta->streamBackend = streamBackendInit(streamPath);
   pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend);
+  pMeta->pTaskBackendUnique =
+      taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
 
   taosMemoryFree(streamPath);
 
   taosInitRWLatch(&pMeta->lock);
+  taosThreadMutexInit(&pMeta->backendMutex, NULL);
+
   return pMeta;
 
 _err:
@@ -136,6 +147,8 @@ void streamMetaClose(SStreamMeta* pMeta) {
   taosRemoveRef(streamBackendId, pMeta->streamBackendRid);
   pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList);
   taosMemoryFree(pMeta->path);
+  taosThreadMutexDestroy(&pMeta->backendMutex);
+  taosHashCleanup(pMeta->pTaskBackendUnique);
   taosMemoryFree(pMeta);
 }
 

source/libs/stream/src/streamState.c

@@ -116,16 +116,33 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
 
   pState->taskId = pStreamTask->id.taskId;
   pState->streamId = pStreamTask->id.streamId;
+  sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId);
 
 #ifdef USE_ROCKSDB
   SStreamMeta* pMeta = pStreamTask->pMeta;
   pState->streamBackendRid = pMeta->streamBackendRid;
+  // taosWLockLatch(&pMeta->lock);
+  taosThreadMutexLock(&pMeta->backendMutex);
+  void* uniqueId =
+      taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
+  if (uniqueId == NULL) {
     int code = streamStateOpenBackend(pMeta->streamBackend, pState);
     if (code == -1) {
-    taosReleaseRef(streamBackendId, pMeta->streamBackendRid);
+      taosReleaseRef(streamBackendId, pState->streamBackendRid);
+      taosThreadMutexUnlock(&pMeta->backendMutex);
       taosMemoryFree(pState);
-    pState = NULL;
+      return NULL;
     }
+    taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1,
+                &pState->pTdbState->backendWrapperId, sizeof(pState->pTdbState->backendWrapperId));
+  } else {
+    int64_t id = *(int64_t*)uniqueId;
+    pState->pTdbState->backendWrapperId = id;
+    pState->pTdbState->pBackendWrapper = taosAcquireRef(streamBackendWrapperId, id);
+
+    taosAcquireRef(streamBackendId, pState->streamBackendRid);
+  }
+  taosThreadMutexUnlock(&pMeta->backendMutex);
 
   pState->pTdbState->pOwner = pTask;
   pState->pFileState = NULL;
@@ -1113,9 +1130,7 @@ int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) {
 #endif
 }
 
-void streamStateReloadInfo(SStreamState* pState, TSKEY ts) {
-  streamFileStateReloadInfo(pState->pFileState, ts);
-}
+void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); }
 
 #if 0
 char* streamStateSessionDump(SStreamState* pState) {