Merge branch 'feature/tq' into 3.0

include/server/vnode/tq/tq.h

@@ -19,9 +19,6 @@
 #include "os.h"
 #include "tutil.h"
 
-#define TQ_ACTION_INSERT 0x7f7f7f7fULL
-#define TQ_ACTION_DELETE 0x80808080ULL
-
 #ifdef __cplusplus
 extern "C" {
 #endif

source/dnode/vnode/tq/CMakeLists.txt

@@ -12,3 +12,7 @@ target_link_libraries(
     PUBLIC os
     PUBLIC util
 )
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/dnode/vnode/tq/inc/tqMetaStore.h

@@ -19,6 +19,11 @@
 #include "os.h"
 #include "tq.h"
 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define TQ_BUCKET_SIZE 0xFF
 #define TQ_PAGE_SIZE 4096
 //key + offset + size
@@ -32,9 +37,23 @@ inline static int TqEmptyTail() { //16
   return TQ_PAGE_SIZE - TqMaxEntryOnePage();
 }
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+#define TQ_ACTION_CONST      0
+#define TQ_ACTION_INUSE      1
+#define TQ_ACTION_INUSE_CONT 2
+#define TQ_ACTION_INTXN      3
+
+#define TQ_SVER              0
+
+static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
+#define TQ_DELETE_TOKEN  (void*)&TQ_CONST_DELETE
+
+typedef struct TqSerializedHead {
+  int16_t ver;
+  int16_t action;
+  int32_t checksum;
+  int64_t ssize;
+  char    content[];
+} TqSerializedHead;
 
 typedef struct TqMetaHandle {
   int64_t key;
@@ -59,30 +78,33 @@ typedef struct TqMetaStore {
   TqMetaList* unpersistHead;
   int fileFd; //TODO:temporaral use, to be replaced by unified tfile
   int idxFd;  //TODO:temporaral use, to be replaced by unified tfile
-  int (*serializer)(TqGroupHandle*, void**);
-  const void* (*deserializer)(const void*, TqGroupHandle*);
+  char* dirPath;
+  int (*serializer)(const void* pObj, TqSerializedHead** ppHead);
+  const void* (*deserializer)(const TqSerializedHead* pHead, void** ppObj);
   void  (*deleter)(void*);
 } TqMetaStore;
 
 TqMetaStore*  tqStoreOpen(const char* path,
-    int serializer(TqGroupHandle*, void**),
-    const void* deserializer(const void*, TqGroupHandle*),
-    void deleter(void*));
+    int serializer(const void* pObj, TqSerializedHead** ppHead),
+    const void* deserializer(const TqSerializedHead* pHead, void** ppObj),
+    void deleter(void* pObj));
 int32_t       tqStoreClose(TqMetaStore*);
 //int32_t       tqStoreDelete(TqMetaStore*);
 //int32_t       TqStoreCommitAll(TqMetaStore*);
 int32_t       tqStorePersist(TqMetaStore*);
 
-TqMetaHandle* tqHandleGet(TqMetaStore*, int64_t key);
-int32_t       tqHandlePut(TqMetaStore*, int64_t key, void* value);
+void*   tqHandleGet(TqMetaStore*, int64_t key);
+int32_t tqHandleMovePut(TqMetaStore*, int64_t key, void* value);
+int32_t tqHandleCopyPut(TqMetaStore*, int64_t key, void* value, size_t vsize);
 //do commit
-int32_t       tqHandleCommit(TqMetaStore*, int64_t key);
+int32_t tqHandleCommit(TqMetaStore*, int64_t key);
 //delete uncommitted
-int32_t       tqHandleAbort(TqMetaStore*, int64_t key);
-//delete committed
-int32_t       tqHandleDel(TqMetaStore*, int64_t key);
+int32_t tqHandleAbort(TqMetaStore*, int64_t key);
+//delete committed kv pair
+//notice that a delete action still needs to be committed
+int32_t tqHandleDel(TqMetaStore*, int64_t key);
 //delete both committed and uncommitted
-int32_t       tqHandleClear(TqMetaStore*, int64_t key);
+int32_t tqHandleClear(TqMetaStore*, int64_t key);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/tq/test/CMakeLists.txt

+add_executable(tqTest "")
+target_sources(tqTest
+    PRIVATE
+    "tqMetaTest.cpp"
+)
+target_include_directories(tqTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/vnode/tq"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+
+target_link_libraries(tqTest
+    tq
+    gtest_main
+)
+enable_testing()
+add_test(
+    NAME tq_test
+    COMMAND tqTest
+)

source/dnode/vnode/tq/test/tqMetaTest.cpp

+#include <gtest/gtest.h>
+#include <cstring>
+#include <iostream>
+#include <queue>
+
+#include "tqMetaStore.h"
+
+struct Foo {
+  int32_t a;
+};
+
+int FooSerializer(const void* pObj, TqSerializedHead** ppHead) {
+  Foo* foo = (Foo*) pObj;
+  if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) {
+    *ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t));
+    (*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t);
+  }
+  *(int32_t*)(*ppHead)->content = foo->a;
+  return (*ppHead)->ssize;
+}
+
+const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) {
+  if(*ppObj == NULL) {
+    *ppObj = realloc(*ppObj, sizeof(int32_t));
+  }
+  Foo* pFoo = *(Foo**)ppObj;
+  pFoo->a = *(int32_t*)pHead->content; 
+  return NULL;
+}
+
+void FooDeleter(void* pObj) {
+  free(pObj); 
+}
+
+class TqMetaTest : public ::testing::Test {
+  protected:
+
+    void SetUp() override {
+      taosRemoveDir(pathName);
+      pMeta = tqStoreOpen(pathName,
+          FooSerializer, FooDeserializer, FooDeleter);
+      ASSERT(pMeta);
+    }
+
+    void TearDown() override {
+      tqStoreClose(pMeta);
+    }
+
+    TqMetaStore* pMeta;
+    const char* pathName = "/tmp/tq_test";
+};
+
+TEST_F(TqMetaTest, copyPutTest) {
+  Foo foo;
+  foo.a = 3;
+  tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo));
+
+  Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+}
+
+TEST_F(TqMetaTest, persistTest) {
+  Foo* pFoo = (Foo*)malloc(sizeof(Foo));
+  pFoo->a = 2;
+  tqHandleMovePut(pMeta, 1, pFoo);
+  Foo* pBar = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pBar == NULL, true);
+  tqHandleCommit(pMeta, 1);
+  pBar = (Foo*)tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pBar->a, pFoo->a);
+  pBar = (Foo*)tqHandleGet(pMeta, 2);
+  EXPECT_EQ(pBar == NULL, true);
+
+  tqStoreClose(pMeta);
+  pMeta = tqStoreOpen(pathName,
+      FooSerializer, FooDeserializer, FooDeleter);
+  ASSERT(pMeta);
+
+  pBar = (Foo*)tqHandleGet(pMeta, 1);
+  ASSERT_EQ(pBar != NULL, true);
+  EXPECT_EQ(pBar->a, 2);
+
+  pBar = (Foo*)tqHandleGet(pMeta, 2);
+  EXPECT_EQ(pBar == NULL, true);
+
+  //taosRemoveDir(pathName);
+}
+
+TEST_F(TqMetaTest, uncommittedTest) {
+  Foo* pFoo = (Foo*)malloc(sizeof(Foo));
+  pFoo->a = 3;
+  tqHandleMovePut(pMeta, 1, pFoo);
+
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+}
+
+TEST_F(TqMetaTest, abortTest) {
+  Foo* pFoo = (Foo*)malloc(sizeof(Foo));
+  pFoo->a = 3;
+  tqHandleMovePut(pMeta, 1, pFoo);
+
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+
+  tqHandleAbort(pMeta, 1);
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+}
+
+TEST_F(TqMetaTest, deleteTest) {
+  Foo* pFoo = (Foo*)malloc(sizeof(Foo));
+  pFoo->a = 3;
+  tqHandleMovePut(pMeta, 1, pFoo);
+
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+
+  tqHandleCommit(pMeta, 1);
+
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  ASSERT_EQ(pFoo != NULL, true);
+  EXPECT_EQ(pFoo->a, 3);
+
+  tqHandleDel(pMeta, 1);
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  ASSERT_EQ(pFoo != NULL, true);
+  EXPECT_EQ(pFoo->a, 3);
+
+  tqHandleCommit(pMeta, 1);
+  pFoo = (Foo*) tqHandleGet(pMeta, 1);
+  EXPECT_EQ(pFoo == NULL, true);
+}

source/os/src/osDir.c

@@ -55,7 +55,7 @@ void taosRemoveDir(const char *dirname) {
   closedir(dir);
   rmdir(dirname);
 
-  printf("dir:%s is removed", dirname);
+  printf("dir:%s is removed\n", dirname);
 }
 
 bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; }
@@ -138,4 +138,4 @@ bool taosRealPath(char *dirname, int32_t maxlen) {
   return true;
 }
 
-#endif
\ No newline at end of file
+#endif