Merge pull request #4886 from taosdata/feature/sim

[TD-2677]<fix>: setting quorum to 2 may cause a deadlock

src/client/src/tscSQLParser.c

@@ -6022,9 +6022,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) {
   }
 
   if (pCreate->quorum != -1 &&
-      (pCreate->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCreate->quorum > TSDB_MAX_DB_REPLICA_OPTION)) {
+      (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) {
     snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum,
-             TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+             TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 

src/common/src/tglobal.c

@@ -811,8 +811,8 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsQuorum;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
-  cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
+  cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION;
+  cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);

src/dnode/src/dnodeVWrite.c

@@ -204,11 +204,12 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
       dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite,
              pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
 
-      pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
+      pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, pWrite);
       if (pWrite->code <= 0) pWrite->processedCount = 1;
+      if (pWrite->code > 0) pWrite->code = 0;
       if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
 
-      dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
+      dTrace("msg:%p is processed in vwrite queue, code:0x%x", pWrite, pWrite->code);
     }
 
     walFsync(vnodeGetWal(pVnode), forceFsync);

src/inc/taosdef.h

@@ -395,6 +395,9 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 #define TSDB_MIN_DB_REPLICA_OPTION      1
 #define TSDB_MAX_DB_REPLICA_OPTION      3
 #define TSDB_DEFAULT_DB_REPLICA_OPTION  1
+
+#define TSDB_MIN_DB_QUORUM_OPTION       1
+#define TSDB_MAX_DB_QUORUM_OPTION       2
 #define TSDB_DEFAULT_DB_QUORUM_OPTION   1
 
 #define TSDB_MAX_JOIN_TABLE_NUM         5

src/mnode/src/mnodeDb.c

@@ -316,9 +316,14 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
-  if (pCfg->quorum < TSDB_MIN_DB_REPLICA_OPTION || pCfg->quorum > TSDB_MAX_DB_REPLICA_OPTION) {
-    mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_REPLICA_OPTION,
-           TSDB_MAX_DB_REPLICA_OPTION);
+  if (pCfg->quorum > pCfg->replications) {
+    mError("invalid db option quorum:%d larger than replica:%d", pCfg->quorum, pCfg->replications);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
+  if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) {
+    mError("invalid db option quorum:%d valid range: [%d, %d]", pCfg->quorum, TSDB_MIN_DB_QUORUM_OPTION,
+           TSDB_MAX_DB_QUORUM_OPTION);
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 

src/vnode/src/vnodeWrite.c

@@ -52,7 +52,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
   int32_t    code = 0;
   SVnodeObj *pVnode = vparam;
   SWalHead * pHead = wparam;
-  SRspRet *  pRspRet = rparam;
+  SVWriteMsg*pWrite = rparam;
+
+  SRspRet *pRspRet = NULL;
+  if (pWrite != NULL) pRspRet = &pWrite->rspRet;
 
   if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
     vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId,
@@ -85,7 +88,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
 
   // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
   int32_t syncCode = 0;
-  syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype);
+  syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype);
   if (syncCode < 0) return syncCode;
 
   // write into WAL

tests/pytest/client/alterDatabase.py

@@ -35,10 +35,9 @@ class TDTestCase:
         tdSql.execute("alter database db keep 365,365,365")
         tdSql.query("show databases")
         tdSql.checkData(0, 7, "365,365,365")
-
-        tdSql.execute("alter database db quorum 2")
-        tdSql.query("show databases")
-        tdSql.checkData(0, 5, 2)
+        
+        tdSql.error("alter database db quorum 2")
+        
 
         tdSql.execute("alter database db blocks 100")
         tdSql.query("show databases")

tests/script/issue/TD-2677.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+
+print ============== deploy
+
+system sh/exec.sh -n dnode1 -s start 
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start 
+system sh/exec.sh -n dnode3 -s start 
+
+print  =============== step1
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+$mnode1Role = $data2_1
+print mnode1Role $mnode1Role
+$mnode2Role = $data2_2
+print mnode2Role $mnode2Role
+$mnode3Role = $data2_3
+print mnode3Role $mnode3Role
+
+if $mnode1Role != master then
+  goto step1
+endi
+if $mnode2Role != slave then
+  goto step1
+endi
+if $mnode3Role != slave then
+  goto step1
+endi
+
+$x = 1
+show2:
+
+print  =============== step1
+sql create database d1 replica 2 quorum 2
+sql create table d1.t1 (ts timestamp, i int)
+sql_error create table d1.t1 (ts timestamp, i int)
+sql insert into d1.t1 values(now, 1)
+sql select * from d1.t1;
+if $rows != 1 then
+  return -1
+endi
+
+print  =============== step2
+sql create database d2 replica 3 quorum 2
+sql create table d2.t1 (ts timestamp, i int)
+sql_error create table d2.t1 (ts timestamp, i int)
+sql insert into d2.t1 values(now, 1)
+sql select * from d2.t1;
+if $rows != 1 then
+  return -1
+endi
+
+print  =============== step3
+sql       create database d4 replica 1 quorum 1
+sql_error create database d5 replica 1 quorum 2
+sql_error create database d6 replica 1 quorum 3
+sql_error create database d7 replica 1 quorum 4
+sql_error create database d8 replica 1 quorum 0
+sql       create database d9 replica 2 quorum 1
+sql       create database d10 replica 2 quorum 2
+sql_error create database d11 replica 2 quorum 3
+sql_error create database d12 replica 2 quorum 4
+sql_error create database d12 replica 2 quorum 0
+sql       create database d13 replica 3 quorum 1
+sql       create database d14 replica 3 quorum 2
+sql_error create database d15 replica 3 quorum 3
+sql_error create database d16 replica 3 quorum 4
+sql_error create database d17 replica 3 quorum 0
+
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT

tests/script/jenkins/basic.txt

 cd ../../../debug; cmake ..
 cd ../../../debug; make
 
+./test.sh -f issue/TD-2677.sim
 ./test.sh -f issue/TD-2680.sim
 ./test.sh -f issue/TD-2713.sim
 

tests/script/tmp/mnodes.sim

@@ -35,7 +35,6 @@ system sh/cfg.sh -n dnode3 -c replica -v 3
 print ============== deploy
 
 system sh/exec.sh -n dnode1 -s start 
-sleep 5001
 sql connect
 
 sql create dnode $hostname2
@@ -45,13 +44,29 @@ system sh/exec.sh -n dnode3 -s start
 
 print  =============== step1
 $x = 0
-show1: 
+step1: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 5 then
-	  return -1
+	sleep 1000
+	if $x == 10 then
+		return -1
 	endi
-sql show mnodes -x show1
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
 $mnode1Role = $data2_1
 print mnode1Role $mnode1Role
 $mnode2Role = $data2_2
@@ -60,13 +75,13 @@ $mnode3Role = $data2_3
 print mnode3Role $mnode3Role
 
 if $mnode1Role != master then
-  goto show1
+  goto step1
 endi
 if $mnode2Role != slave then
-  goto show1
+  goto step1
 endi
 if $mnode3Role != slave then
-  goto show1
+  goto step1
 endi
 
 $x = 1
@@ -75,9 +90,21 @@ show2:
 print  =============== step $x
 sql show mnodes
 print $data0_1 $data2_1
-print $data0_2 $data2_2
-print $data0_3 $data2_3
 
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
 
 $x = $x + 1
 sleep 5000