Merge branch 'master' into develop

# Conflicts:
#	src/client/src/tscServer.c

packaging/docker/Dockerfile

+FROM centos:7
+
+WORKDIR /root
+
+COPY tdengine.tar.gz /root/
+RUN tar -zxf tdengine.tar.gz
+WORKDIR /root/tdengine/
+RUN sh install.sh
+
+
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+EXPOSE 6020 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042
+EXPOSE 6043 6044 6045 6046 6047 6048 6049 6050
+CMD ["taosd"]
+VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]

packaging/docker/dockerbuild.sh

+#!/bin/bash
+set -x
+$1
+tar -zxf $1
+DIR=`echo $1|awk -F . '{print($1"."$2"."$3"."$4)}'`
+mv $DIR tdengine
+tar -czf tdengine.tar.gz tdengine
+TMP=`echo $1|awk -F . '{print($2"."$3"."$4)}'`
+TAG="1."$TMP
+docker build --rm -f "Dockerfile" -t tdengine/tdengine:$TAG "."
+docker login -u tdengine -p ********  #replace the docker registry username and password
+docker push tdengine/tdengine:$TAG
\ No newline at end of file

src/inc/taoserror.h

@@ -91,7 +91,7 @@ extern "C" {
 #define TSDB_CODE_INVALID_QHANDLE            70
 #define TSDB_CODE_RELATED_TABLES_EXIST       71
 #define TSDB_CODE_MONITOR_DB_FORBEIDDEN      72
-#define TSDB_CODE_VG_COMMITLOG_INIT_FAILED   73
+#define TSDB_CODE_NO_DISK_PERMISSIONS        73
 #define TSDB_CODE_VG_INIT_FAILED             74
 #define TSDB_CODE_DATA_ALREADY_IMPORTED      75
 #define TSDB_CODE_OPS_NOT_SUPPORT            76

src/modules/http/src/gcJson.c

@@ -79,7 +79,9 @@ void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
   if (jsonBuf == NULL) return;
 
   // write end of target
-  gcWriteTargetEndJson(jsonBuf);
+  if (cmd->numOfRows != 0) {
+    gcWriteTargetEndJson(jsonBuf);
+  }
 }
 
 bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) {
@@ -116,8 +118,8 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
 
   if (groupFields == -1 && cmd->numOfRows == 0) {
     gcWriteTargetStartJson(jsonBuf, refIdBuffer, aliasBuffer);
-    cmd->numOfRows += numOfRows;
   }
+  cmd->numOfRows += numOfRows;
 
   for (int k = 0; k < numOfRows; ++k) {
     TAOS_ROW row = taos_fetch_row(result);
@@ -158,7 +160,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
           }
           break;
         default:
-          snprintf(target, HTTP_GC_TARGET_SIZE, "%s%s", aliasBuffer, "invalidcol");
+          snprintf(target, HTTP_GC_TARGET_SIZE, "%s%s", aliasBuffer, "-");
           break;
       }
 
@@ -217,7 +219,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
           }
           break;
         default:
-          httpJsonString(jsonBuf, "invalidcol", 10);
+          httpJsonString(jsonBuf, "-", 1);
           break;
       }
     }

src/modules/http/src/httpJson.c

@@ -310,7 +310,9 @@ void httpJsonInt(JsonBuf* buf, int num) {
 void httpJsonFloat(JsonBuf* buf, float num) {
   httpJsonItemToken(buf);
   httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
-  if (num > 1E10 || num < -1E10) {
+  if (isinf(num) || isnan(num)) {
+    buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null");
+  } else if (num > 1E10 || num < -1E10) {
     buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5e", num);
   } else {
     buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5f", num);
@@ -320,7 +322,9 @@ void httpJsonFloat(JsonBuf* buf, float num) {
 void httpJsonDouble(JsonBuf* buf, double num) {
   httpJsonItemToken(buf);
   httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
-  if (num > 1E10 || num < -1E10) {
+  if (isinf(num) || isnan(num)) {
+    buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null");
+  } else if (num > 1E10 || num < -1E10) {
     buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9e", num);
   } else {
     buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9f", num);

src/modules/http/src/restHandle.c

@@ -67,10 +67,16 @@ bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) {
     return false;
   }
 
+
+  /*
+   * for async test
+   * /
+  /*
   if (httpCheckUsedbSql(sql)) {
     httpSendErrorResp(pContext, HTTP_NO_EXEC_USEDB);
     return false;
   }
+  */
 
   HttpSqlCmd* cmd = &(pContext->singleCmd);
   cmd->nativSql = sql;

src/rpc/src/tstring.c

@@ -197,7 +197,7 @@ char *tsError[] = {"success",
                    "invalid query handle",         // 70
                    "tables related to metric exist",
                    "can't drop monitor database or tables",
-                   "commit log init failed",
+                   "no disk permissions",
                    "vgroup init failed",
                    "data is already imported",     // 75
                    "not supported operation",
@@ -234,7 +234,7 @@ char *tsError[] = {"success",
                    "timestamp disordered in cache block",
                    "timestamp disordered in file block",
                    "invalid commit log",
-                   "server no disk space",    //110
+                   "no disk space on server",    //110
                    "only super table has metric meta info",
                    "tags value not unique for join",
                    "invalid submit message",

src/system/detail/inc/vnodeStatus.h

@@ -16,16 +16,20 @@
 #ifndef TDENGINE_TSTATUS_H
 #define TDENGINE_TSTATUS_H
 
+#include "taoserror.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 enum _TSDB_VG_STATUS {
-  TSDB_VG_STATUS_READY,
-  TSDB_VG_STATUS_IN_PROGRESS,
-  TSDB_VG_STATUS_COMMITLOG_INIT_FAILED,
-  TSDB_VG_STATUS_INIT_FAILED,
-  TSDB_VG_STATUS_FULL
+  TSDB_VG_STATUS_READY = TSDB_CODE_SUCCESS,
+  TSDB_VG_STATUS_IN_PROGRESS = TSDB_CODE_ACTION_IN_PROGRESS,
+  TSDB_VG_STATUS_NO_DISK_PERMISSIONS = TSDB_CODE_NO_DISK_PERMISSIONS,
+  TSDB_VG_STATUS_SERVER_NO_PACE = TSDB_CODE_SERVER_NO_SPACE,
+  TSDB_VG_STATUS_SERV_OUT_OF_MEMORY = TSDB_CODE_SERV_OUT_OF_MEMORY,
+  TSDB_VG_STATUS_INIT_FAILED = TSDB_CODE_VG_INIT_FAILED,
+  TSDB_VG_STATUS_FULL = TSDB_CODE_NO_ENOUGH_DNODES,
 };
 
 enum _TSDB_DB_STATUS {

src/system/detail/src/dnodeMgmt.c

@@ -415,10 +415,10 @@ int vnodeProcessVPeerCfgRsp(char *msg, int msgLen, SMgmtObj *pMgmtObj) {
     int32_t *pint = (int32_t *)pRsp->more;
     int      vnode = htonl(*pint);
     if (vnode < TSDB_MAX_VNODES && vnodeList[vnode].lastKey != 0) {
-      dError("vnode:%d not configured, it shall be empty");
+      dError("vnode:%d not configured, it shall be empty, code:%d", vnode, pRsp->code);
       vnodeRemoveVnode(vnode);
     } else {
-      dTrace("vnode:%d is invalid", vnode);
+      dError("vnode:%d is invalid, code:%d", vnode, pRsp->code);
     }
   }
 

src/system/detail/src/mgmtDnodeInt.c

@@ -152,19 +152,15 @@ int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) {
     return 0;
   }
 
-  if (pRsp->code == 0) {
+  if (pRsp->code == TSDB_CODE_SUCCESS) {
     pDb->vgStatus = TSDB_VG_STATUS_READY;
     mTrace("dnode:%s, db:%s vgroup is created in dnode", taosIpStr(pObj->privateIp), pRsp->more);
     return 0;
   }
 
-  if (pRsp->code == TSDB_CODE_VG_COMMITLOG_INIT_FAILED) {
-    pDb->vgStatus = TSDB_VG_STATUS_COMMITLOG_INIT_FAILED;
-    mError("dnode:%s, db:%s vgroup commit log init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code);
-  } else {
-    pDb->vgStatus = TSDB_VG_STATUS_INIT_FAILED;
-    mError("dnode:%s, db:%s vgroup init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code);
-  }
+  pDb->vgStatus = pRsp->code;
+  mError("dnode:%s, db:%s vgroup init failed, code:%d %s",
+          taosIpStr(pObj->privateIp), pRsp->more, pRsp->code, taosGetVgroupStatusStr(pDb->vgStatus));
 
   return 0;
 }

src/system/detail/src/mgmtMeter.c

@@ -657,16 +657,13 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) {
       return TSDB_CODE_NO_ENOUGH_DNODES;
     }
 
-    if (pDb->vgStatus == TSDB_VG_STATUS_COMMITLOG_INIT_FAILED) {
+    if (pDb->vgStatus == TSDB_VG_STATUS_NO_DISK_PERMISSIONS ||
+        pDb->vgStatus == TSDB_VG_STATUS_SERVER_NO_PACE ||
+        pDb->vgStatus == TSDB_VG_STATUS_SERV_OUT_OF_MEMORY ||
+        pDb->vgStatus == TSDB_VG_STATUS_INIT_FAILED ) {
       mgmtDestroyMeter(pMeter);
-      mError("table:%s, commit log init failed", pCreate->meterId);
-      return TSDB_CODE_VG_COMMITLOG_INIT_FAILED;
-    }
-
-    if (pDb->vgStatus == TSDB_VG_STATUS_INIT_FAILED) {
-      mgmtDestroyMeter(pMeter);
-      mError("table:%s, vgroup init failed", pCreate->meterId);
-      return TSDB_CODE_VG_INIT_FAILED;
+      mError("table:%s, vgroup init failed, reason:%d %s", pCreate->meterId, pDb->vgStatus, taosGetVgroupStatusStr(pDb->vgStatus));
+      return pDb->vgStatus;
     }
 
     if (pVgroup == NULL) {

src/system/detail/src/vnodeFile.c

@@ -1823,7 +1823,7 @@ int vnodeUpdateFileMagic(int vnode, int fileId) {
 }
 
 int vnodeInitFile(int vnode) {
-  int        code = 0;
+  int        code = TSDB_CODE_SUCCESS;
   SVnodeObj *pVnode = vnodeList + vnode;
 
   pVnode->maxFiles = pVnode->cfg.daysToKeep / pVnode->cfg.daysPerFile + 1;

src/system/detail/src/vnodeMeter.c

@@ -77,8 +77,14 @@ int vnodeCreateMeterObjFile(int vnode) {
   sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode);
   fp = fopen(fileName, "w+");
   if (fp == NULL) {
-    dError("failed to create vnode:%d file:%s", vnode, fileName);
-    return -1;
+    dError("failed to create vnode:%d file:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
+    if (errno == EACCES) {
+      return TSDB_CODE_NO_DISK_PERMISSIONS;
+    } else if (errno == ENOSPC) {
+      return TSDB_CODE_SERVER_NO_SPACE;
+    } else {
+      return TSDB_CODE_VG_INIT_FAILED;
+    }
   } else {
     vnodeCreateFileHeader(fp);
     vnodeUpdateVnodeFileHeader(fp, vnodeList + vnode);
@@ -93,7 +99,7 @@ int vnodeCreateMeterObjFile(int vnode) {
     fclose(fp);
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 FILE *vnodeOpenMeterObjFile(int vnode) {
@@ -271,7 +277,7 @@ int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
   /* vnodeUpdateFileCheckSum(fp); */
   fclose(fp);
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int vnodeSaveVnodeInfo(int vnode) {
@@ -568,6 +574,14 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
     return code;
   }
 
+  /*
+   * please refer to TBASE-926, data may be lost when the cache is full
+   */
+  if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) {
+    code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion);
+    if (code != TSDB_CODE_SUCCESS) return code;
+  }
+
   SCachePool *pPool = (SCachePool *)pVnode->pCachePool;
   if (pObj->freePoints < numOfPoints || pObj->freePoints < (pObj->pointsPerBlock << 1) ||
       pPool->notFreeSlots > pVnode->cfg.cacheNumOfBlocks.totalBlocks - 2) {
@@ -585,11 +599,6 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
     if (code != TSDB_CODE_SUCCESS) return code;
   }
 
-  if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) {
-    code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion);
-    if (code != TSDB_CODE_SUCCESS) return code;
-  }
-
   if (pObj->sversion < sversion) {
     dTrace("vid:%d sid:%d id:%s, schema is changed, new:%d old:%d", pObj->vnode, pObj->sid, pObj->meterId, sversion,
            pObj->sversion);

src/system/detail/src/vnodeStatus.c

@@ -19,11 +19,13 @@
 
 const char* taosGetVgroupStatusStr(int32_t vgroupStatus) {
   switch (vgroupStatus) {
-    case TSDB_VG_STATUS_READY:                 return "ready";
-    case TSDB_VG_STATUS_IN_PROGRESS:           return "inprogress";
-    case TSDB_VG_STATUS_COMMITLOG_INIT_FAILED: return "commitlog_init_failed";
-    case TSDB_VG_STATUS_INIT_FAILED:           return "init_failed";
-    case TSDB_VG_STATUS_FULL:                  return "full";
+    case TSDB_VG_STATUS_READY:                 return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_IN_PROGRESS:           return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_NO_DISK_PERMISSIONS:   return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_SERVER_NO_PACE:        return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY:    return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_INIT_FAILED:           return tsError[vgroupStatus];
+    case TSDB_VG_STATUS_FULL:                  return tsError[vgroupStatus];
     default:                                   return "undefined";
   }
 }

src/system/detail/src/vnodeStore.c

@@ -42,24 +42,24 @@ static int vnodeInitStoreVnode(int vnode) {
   pVnode->pCachePool = vnodeOpenCachePool(vnode);
   if (pVnode->pCachePool == NULL) {
     dError("vid:%d, cache pool init failed.", pVnode->vnode);
-    return -1;
+    return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
 
-  if (vnodeInitFile(vnode) < 0) {
+  if (vnodeInitFile(vnode) != TSDB_CODE_SUCCESS) {
     dError("vid:%d, files init failed.", pVnode->vnode);
-    return -1;
+    return TSDB_CODE_VG_INIT_FAILED;
   }
 
-  if (vnodeInitCommit(vnode) < 0) {
+  if (vnodeInitCommit(vnode) != TSDB_CODE_SUCCESS) {
     dError("vid:%d, commit init failed.", pVnode->vnode);
-    return -1;
+    return TSDB_CODE_VG_INIT_FAILED;
   }
 
   pthread_mutex_init(&(pVnode->vmutex), NULL);
   dPrint("vid:%d, storage initialized, version:%ld fileId:%d numOfFiles:%d", vnode, pVnode->version, pVnode->fileId,
          pVnode->numOfFiles);
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int vnodeOpenVnode(int vnode) {
@@ -183,22 +183,45 @@ int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) {
   vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_CREATING;
 
   sprintf(fileName, "%s/vnode%d", tsDirectory, vnode);
-  mkdir(fileName, 0755);
+  if (mkdir(fileName, 0755) != 0) {
+    dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
+    if (errno == EACCES) {
+      return TSDB_CODE_NO_DISK_PERMISSIONS;
+    } else if (errno == ENOSPC) {
+      return TSDB_CODE_SERVER_NO_SPACE;
+    } else if (errno == EEXIST) {
+    } else {
+      return TSDB_CODE_VG_INIT_FAILED;
+    }
+  }
 
   sprintf(fileName, "%s/vnode%d/db", tsDirectory, vnode);
-  mkdir(fileName, 0755);
+  if (mkdir(fileName, 0755) != 0) {
+    dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno));
+    if (errno == EACCES) {
+      return TSDB_CODE_NO_DISK_PERMISSIONS;
+    } else if (errno == ENOSPC) {
+      return TSDB_CODE_SERVER_NO_SPACE;
+    } else if (errno == EEXIST) {
+    } else {
+      return TSDB_CODE_VG_INIT_FAILED;
+    }
+  }
 
   vnodeList[vnode].cfg = *pCfg;
-  if (vnodeCreateMeterObjFile(vnode) != 0) {
-    return TSDB_CODE_VG_INIT_FAILED;
+  int code = vnodeCreateMeterObjFile(vnode);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
   }
 
-  if (vnodeSaveVnodeCfg(vnode, pCfg, pDesc) != 0) {
+  code = vnodeSaveVnodeCfg(vnode, pCfg, pDesc);
+  if (code != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_VG_INIT_FAILED;
   }
 
-  if (vnodeInitStoreVnode(vnode) < 0) {
-    return TSDB_CODE_VG_COMMITLOG_INIT_FAILED;
+  code = vnodeInitStoreVnode(vnode);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
   }
 
   return vnodeOpenVnode(vnode);
@@ -291,7 +314,8 @@ int vnodeInitStore() {
   if (vnodeInitInfo() < 0) return -1;
 
   for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
-    if (vnodeInitStoreVnode(vnode) < 0) {
+    int code = vnodeInitStoreVnode(vnode);
+    if (code != TSDB_CODE_SUCCESS) {
       // one vnode is failed to recover from commit log, continue for remain
       return -1;
     }