Merge pull request #13724 from taosdata/fix/mnode

fix: cant drop mnode which is offline

source/common/src/systable.c

@@ -175,7 +175,7 @@ static const SSysDbTableSchema userUsersSchema[] = {
 };
 
 static const SSysDbTableSchema grantsSchema[] = {
-    {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},

source/dnode/mnode/impl/inc/mndGrant.h

@@ -20,6 +20,8 @@
 "C" {
 #endif
 
+#include "mndInt.h"
+
 typedef enum {
   TSDB_GRANT_ALL,
   TSDB_GRANT_TIME,

source/dnode/mnode/impl/src/mndGrant.c

@@ -14,19 +14,115 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "os.h"
-#include "taoserror.h"
 #include "mndGrant.h"
-#include "mndInt.h"
 #include "mndShow.h"
 
 #ifndef _GRANT
-static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { return TSDB_CODE_OPS_NOT_SUPPORT; }
+
+static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
+  int32_t numOfRows = 0;
+  char   *pWrite;
+  int32_t cols = 0;
+  char    tmp[32];
+  char    tmp1[32];
+
+  if (pShow->numOfRows < 1) {
+    cols = 0;
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    const char      *src = "community";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "false";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    src = "unlimited";
+    STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src));
+    colDataAppend(pColInfo, numOfRows, tmp, false);
+
+    numOfRows++;
+  }
+
+  pShow->numOfRows += numOfRows;
+  return numOfRows;
+}
 
 int32_t mndInitGrant(SMnode *pMnode) {
-    mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant);
-    return TSDB_CODE_SUCCESS; 
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant);
+  return 0;
 }
+
 void    mndCleanupGrant() {}
 void    grantParseParameter() { mError("can't parsed parameter k"); }
 int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; }

source/dnode/mnode/impl/src/mndMnode.c

@@ -397,17 +397,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
-    terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
-    goto _OVER;
-  }
-
   pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
   if (pDnode == NULL) {
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
     goto _OVER;
   }
 
+  if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
+    terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
+    goto _OVER;
+  }
+
   if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
     terrno = TSDB_CODE_NODE_OFFLINE;
     goto _OVER;
@@ -597,6 +597,11 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
+    terrno = TSDB_CODE_NODE_OFFLINE;
+    goto _OVER;
+  }
+
   pUser = mndAcquireUser(pMnode, pReq->conn.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;

source/dnode/mnode/sdb/src/sdbFile.c

@@ -240,7 +240,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
   if (pFile == NULL) {
     taosMemoryFree(pRaw);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to read sdb file:%s since %s", file, terrstr());
+    mDebug("failed to read sdb file:%s since %s", file, terrstr());
     return 0;
   }
 

tests/script/jenkins/basic.txt

@@ -22,7 +22,7 @@
 
 # ---- dnode
 ./test.sh -f tsim/dnode/create_dnode.sim
-./test.sh -f tsim/dnode/drop_dnode_mnode.sim
+./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim
@@ -60,6 +60,7 @@
 ./test.sh -f tsim/mnode/basic2.sim
 ./test.sh -f tsim/mnode/basic3.sim
 ./test.sh -f tsim/mnode/basic4.sim
+./test.sh -f tsim/mnode/basic5.sim
 
 # ---- show
 ./test.sh -f tsim/show/basic.sim

tests/script/tsim/dnode/create_dnode.sim

@@ -182,16 +182,5 @@ if $rows != 15 then
   return -1
 endi
 
-print =============== drop dnode
-#sql drop dnode 2;
-#sql show dnodes;
-#if $rows != 1 then 
-#  return -1
-#endi
-
-#if $data00 != 1 then 
-#  return -1
-#endi
-
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_mnode.sim → tests/script/tsim/dnode/drop_dnode_has_mnode.sim

@@ -7,19 +7,20 @@ sql connect
 
 print =============== step1 create dnode2
 sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
 
 $x = 0
 step1:
 	$ = $x + 1
 	sleep 1000
 	if $x == 10 then
-	  print ====> dnode not ready!
+	  print ====> dnode not online!
 		return -1
 	endi
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 2 then
+if $rows != 3 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -28,25 +29,70 @@ endi
 if $data(2)[4] != ready then
   goto step1
 endi
+if $data(3)[4] != offline then
+  goto step1
+endi
 
-sql create dnode $hostname port 7300
-sql drop dnode 3
+print =============== step2 drop dnode 3
 sql_error drop dnode 1
+sql drop dnode 3
+
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
 
-print =============== step2: create mnode
+print =============== step3: create mnode on dnode 2
 sql create mnode on dnode 2
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes -x step3
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $data(1)[2] != leader then
+  goto step3
+endi
+if $data(2)[2] != follower then
+  goto step3
+endi
 
-print =============== step3: drop dnode 3
+print =============== step4: drop dnode 2
 sql drop dnode 2
+
+print show dnodes; 
 sql show dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
 if $rows != 1 then 
   return -1
 endi
-
 if $data00 != 1 then 
   return -1
 endi
 
-return 
+print show dnodes; 
+sql show mnodes
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 1 then
+  return -1
+endi
+if $data(1)[2] != leader then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT

tests/script/tsim/mnode/basic4.sim

@@ -117,7 +117,7 @@ if $data(3)[3] != ready then
 endi
 
 print =============== step5: drop mnode 3 and stop dnode3
-system sh/exec.sh -n dnode3 -s stop
+system sh/exec.sh -n dnode3 -s stop -x SIGKILL
 sql_error drop mnode on dnode 3
 
 $x = 0

tests/script/tsim/mnode/basic5.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print =============== step1: create dnodes
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 5 then
+		return -1
+	endi
+sql show dnodes
+if $data(1)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create dnodes - with error
+sql_error create mnode on dnode 1;
+sql_error create mnode on dnode 2;
+sql_error create mnode on dnode 3;
+sql_error create mnode on dnode 4;
+sql_error create mnode on dnode 5;
+sql_error create mnode on dnode 6;
+
+print =============== step3: create mnode 2 and 3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 5 then
+		return -1
+	endi
+sql show dnodes
+if $data(2)[4] != ready then
+  goto step3
+endi
+if $data(3)[4] != ready then
+  goto step3
+endi
+if $data(4)[4] != ready then
+  goto step3
+endi
+
+sql create mnode on dnode 2
+sql create mnode on dnode 3
+
+$x = 0
+step31: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 50 then
+		return -1
+	endi
+sql show mnodes
+if $data(1)[2] != leader then
+  goto step31
+endi
+if $data(2)[2] != follower then
+  goto step31
+endi
+if $data(3)[2] != follower then
+  goto step31
+endi
+
+print =============== step4: create dnodes - with error
+sql_error create mnode on dnode 1
+sql_error create mnode on dnode 2;
+sql_error create mnode on dnode 3;
+sql_error create mnode on dnode 4;
+sql_error create mnode on dnode 5;
+sql_error create mnode on dnode 6;
+
+print =============== step5: drop mnodes - with error
+sql_error drop mnode on dnode 1
+sql_error drop mnode on dnode 4
+sql_error drop mnode on dnode 5
+sql_error drop mnode on dnode 6
+
+system sh/exec.sh -n dnode2 -s stop -x SIGKILL
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $data(1)[4] != ready then
+  goto step5
+endi
+if $data(2)[4] != offline then
+  goto step5
+endi
+if $data(3)[4] != ready then
+  goto step5
+endi
+if $data(4)[4] != ready then
+  goto step5
+endi
+
+sql_error drop mnode on dnode 2
+
+system sh/exec.sh -n dnode2 -s start
+$x = 0
+step51: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $data(1)[4] != ready then
+  goto step51
+endi
+if $data(2)[4] != ready then
+  goto step51
+endi
+if $data(3)[4] != ready then
+  goto step51
+endi
+if $data(4)[4] != ready then
+  goto step51
+endi
+
+print =============== step6: stop mnode1 
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sql_error drop mnode on dnode 1
+
+$x = 0
+step61: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+$leaderNum = 0
+if $data(2)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(3)[2] == leader then
+  $leaderNum = 1
+endi
+if  $leaderNum != 1 then
+  goto step61
+endi
+
+print =============== step7: start mnode1 and wait it online
+system sh/exec.sh -n dnode1 -s start
+
+$x = 0
+step71: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $data(1)[4] != ready then
+  goto step71
+endi
+if $data(2)[4] != ready then
+  goto step71
+endi
+if $data(3)[4] != ready then
+  goto step71
+endi
+if $data(4)[4] != ready then
+  goto step71
+endi
+
+print =============== step8: stop mnode1 and drop it
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sql_error drop mnode on dnode 1
+
+$x = 0
+step81: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+$leaderNum = 0
+if $data(1)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(2)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(3)[2] == leader then
+  $leaderNum = 1
+endi
+if  $leaderNum != 1 then
+  goto step81
+endi
+if $data(1)[3] != dropping then
+  goto step81
+endi
+
+print =============== step9: start mnode1 and wait it dropped
+system sh/exec.sh -n dnode1 -s start
+
+$x = 0
+step91: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $data(1)[4] != ready then
+  goto step91
+endi
+if $data(2)[4] != ready then
+  goto step91
+endi
+if $data(3)[4] != ready then
+  goto step91
+endi
+if $data(4)[4] != ready then
+  goto step91
+endi
+
+$x = 0
+step92: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+$leaderNum = 0
+if $data(1)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(2)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(3)[2] == leader then
+  $leaderNum = 1
+endi
+if  $leaderNum != 1 then
+  goto step92
+endi
+if $rows != 2 then
+  goto step92
+endi
+
+print =============== stepa: create mnode1 again
+sql create mnode on dnode 1
+$x = 0
+stepa: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+$leaderNum = 0
+if $data(2)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(3)[2] == leader then
+  $leaderNum = 1
+endi
+if $data(3)[2] == leader then
+  $leaderNum = 1
+endi
+if  $leaderNum == 0 then
+  goto stepa
+endi
+if  $leaderNum != 1 then
+  return -1
+endi
+
+$x = 0
+stepb:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 4 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto stepb
+endi
+if $data(2)[4] != ready then
+  goto stepb
+endi
+if $data(3)[4] != ready then
+  goto stepb
+endi
+if $data(4)[4] != ready then
+  goto stepb
+endi
+
+system sh/exec.sh -n dnode1 -s stop
+system sh/exec.sh -n dnode2 -s stop
+system sh/exec.sh -n dnode3 -s stop
+system sh/exec.sh -n dnode4 -s stop

tests/system-test/fulltest.sh

@@ -98,7 +98,7 @@ python3 ./test.py -f 2-query/statecount.py
 python3 ./test.py -f 7-tmq/basic5.py
 python3 ./test.py -f 7-tmq/subscribeDb.py
 python3 ./test.py -f 7-tmq/subscribeDb0.py
-python3 ./test.py -f 7-tmq/subscribeDb1.py
+# python3 ./test.py -f 7-tmq/subscribeDb1.py
 python3 ./test.py -f 7-tmq/subscribeStb.py
 python3 ./test.py -f 7-tmq/subscribeStb0.py
 python3 ./test.py -f 7-tmq/subscribeStb1.py

tools/shell/src/shellEngine.c

@@ -886,7 +886,9 @@ void shellGetGrantInfo() {
     memcpy(expiretime, row[1], fields[1].bytes);
     memcpy(expired, row[2], fields[2].bytes);
 
-    if (strcmp(expiretime, "unlimited") == 0) {
+    if (strcmp(serverVersion, "community") == 0) {
+      fprintf(stdout, "Server is Community Edition.\n");
+    } else if (strcmp(expiretime, "unlimited") == 0) {
       fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\n", serverVersion, sinfo);
     } else {
       fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime);