TD-1793

src/inc/mnode.h

@@ -47,6 +47,7 @@ typedef struct SMnodeMsg {
   int8_t    successed;
   int8_t    expected;
   int8_t    retry;
+  int32_t   incomingTs;
   int32_t   code;
   void *    pObj;
   SRpcMsg   rpcMsg;

src/inc/taoserror.h

@@ -181,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB,               0, 0x0383, "Invalid da
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN,     0, 0x0384, "Cannot delete monitor database")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES,       0, 0x0385, "Too many databases for account")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database not available")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         0, 0x0387, "Database unsynced")
 
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")

src/kit/shell/inc/shell.h

@@ -60,7 +60,7 @@ typedef struct SShellArguments {
 extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
 extern TAOS* shellInit(SShellArguments* args);
 extern void* shellLoopQuery(void* arg);
-extern void taos_error(TAOS_RES* tres);
+extern void taos_error(TAOS_RES* tres, int64_t st);
 extern int regex_match(const char* s, const char* reg, int cflags);
 void shellReadCommand(TAOS* con, char command[]);
 int32_t shellRunCommand(TAOS* con, char* command);

src/kit/shell/src/shellEngine.c

@@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
 
   TAOS_RES* pSql = taos_query_h(con, command, &result);
   if (taos_errno(pSql)) {
-    taos_error(pSql);
+    taos_error(pSql, st);
     return;
   }
 
@@ -807,9 +807,10 @@ void write_history() {
   fclose(f);
 }
 
-void taos_error(TAOS_RES *tres) {
+void taos_error(TAOS_RES *tres, int64_t st) {
+  int64_t et = taosGetTimestampUs();
   atomic_store_ptr(&result, 0);
-  fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
+  fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6);
   taos_free_result(tres);
 }
 

src/mnode/src/mnodeInt.c

@@ -40,6 +40,7 @@ void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
 
   pMsg->rpcMsg = *pRpcMsg;
   pMsg->rpcMsg.pCont = pMsg->pCont;
+  pMsg->incomingTs = taosGetTimestampSec();
   memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
 
   return pMsg;

src/mnode/src/mnodeTable.c

@@ -45,6 +45,10 @@
 #include "mnodeRead.h"
 #include "mnodePeer.h"
 
+#define ALTER_CTABLE_RETRY_TIMES  3
+#define CREATE_CTABLE_RETRY_TIMES 10
+#define CREATE_CTABLE_RETRY_SEC   10
+
 static void *  tsChildTableSdb;
 static void *  tsSuperTableSdb;
 static int32_t tsChildTableUpdateSize;
@@ -2421,7 +2425,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
       dnodeSendRpcMWriteRsp(mnodeMsg, code);
     }
   } else {
-    if (mnodeMsg->retry++ < 10) {
+    mnodeMsg->retry++;
+    int32_t sec = taosGetTimestampSec();
+    if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
       mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
              " result:%s thandle:%p",
              mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid,
@@ -2429,13 +2435,18 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
 
       dnodeDelayReprocessMWriteMsg(mnodeMsg);
     } else {
-      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
+      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
+             ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d",
              mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
-             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry);
 
       SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
       sdbDeleteRow(&oper);
 
+      if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) {
+        //Avoid retry again in client
+        rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY;
+      }
       dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
     }
   }
@@ -2456,7 +2467,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
 
     dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
   } else {
-    if (mnodeMsg->retry++ < 3) {
+    if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
       mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
              mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
              mnodeMsg->rpcMsg.handle);