Merge branch 'develop' into feature/2.0tsdb

documentation/webdocs/markdowndocs/TAOS SQL-ch.md

@@ -63,11 +63,11 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
 | 3    |  BIGINT   | 8      | 长整型，范围 [-2^63+1,   2^63-1], -2^63用于NULL                                 |
 | 4    |   FLOAT   | 4      | 浮点型，有效位数6-7，范围 [-3.4E38, 3.4E38]                  |
 | 5    |  DOUBLE   | 8      | 双精度浮点型，有效位数15-16，范围 [-1.7E308,   1.7E308]      |
-| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有65526字节，但由于每行数据最多64K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节，将被自动截断。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
+| 6    |  BINARY   | 自定义 | 用于记录字符串，理论上，最长可以有65526字节，但由于每行数据最多64K字节，实际上限一般小于理论值。 binary仅支持字符串输入，字符串两端使用单引号引用，否则英文全部自动转化为小写。使用时须指定大小，如binary(20)定义了最长为20个字符的字符串，每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示， 即 **\’**。 |
 | 7    | SMALLINT  | 2      | 短整型， 范围 [-32767, 32767], -32768用于NULL                                |
 | 8    |  TINYINT  | 1      | 单字节整型，范围 [-127, 127], -128用于NULL                                |
 | 9    |   BOOL    | 1      | 布尔型，{true,   false}                                      |
-| 10   |   NCHAR   | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将被自动截断。 |
+| 10   |   NCHAR   | 自定义 | 用于记录非ASCII字符串，如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小，类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符，会固定占用40bytes的空间。如用户字符串长度超出声明长度，则将会报错。 |
 
 **Tips**: TDengine对SQL语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。
 

documentation/webdocs/markdowndocs/TAOS SQL.md

@@ -39,8 +39,8 @@ The full list of data types is listed below.  For string types of data, we will
 | 6    |   DOUBLE    |    8    | A standard nullable double float type with 15-16 significant digits and a range of [-1.7E308, 1.7E308]​ |
 | 7    |    BOOL     |    1    | A nullable boolean type, [**`true`**, **`false`**]           |
 | 8    |  TIMESTAMP  |    8    | A nullable timestamp type with the same usage as the primary column timestamp |
-| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, any exceeded chars will be automatically truncated, the maximum length of *M* is 65526, but as maximum row size is 64K bytes, the actual upper limit will generally less than 65526. This type of string only supports ASCii encoded chars. |
-| 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, any exceeded chars will be truncated. The **`NCHAR`** type supports Unicode encoded chars. |
+| 9    | BINARY(*M*) |   *M*   | A nullable string type whose length is *M*, error should be threw with exceeded chars, the maximum length of *M* is 65526, but as maximum row size is 64K bytes, the actual upper limit will generally less than 65526. This type of string only supports ASCii encoded chars. |
+| 10   | NCHAR(*M*)  | 4 * *M* | A nullable string type whose length is *M*, error should be threw with exceeded chars. The **`NCHAR`** type supports Unicode encoded chars. |
 
 All the keywords in a SQL statement are case-insensitive, but strings values are case-sensitive and must be quoted by a pair of `'` or `"`. To quote a `'` or a `"` , you can use the escape character `\`.
 

src/common/inc/tglobal.h

@@ -49,7 +49,7 @@ extern int32_t tsTotalMemoryMB;
 extern int32_t tsVersion;
 
 extern int32_t tscEmbedded;
-extern int64_t tsMsPerDay[2];
+extern int64_t tsMsPerDay[3];
 
 extern char  tsFirst[];
 extern char  tsSecond[];

src/common/src/tglobal.c

@@ -55,11 +55,12 @@ int32_t tsEnableCoreFile = 0;
 int32_t tscEmbedded = 0;
 
 /*
- * minmum scale for whole system, millisecond by default
+ * minimum scale for whole system, millisecond by default
  * for TSDB_TIME_PRECISION_MILLI: 86400000L
  *     TSDB_TIME_PRECISION_MICRO: 86400000000L
+ *     TSDB_TIME_PRECISION_NANO:  86400000000000L
  */
-int64_t tsMsPerDay[] = {86400000L, 86400000000L};
+int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
 
 char  tsFirst[TSDB_EP_LEN] = {0};  
 char  tsSecond[TSDB_EP_LEN] = {0};

src/inc/taosdef.h

@@ -92,6 +92,7 @@ extern const int32_t TYPE_BYTES[11];
 
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1
+#define TSDB_TIME_PRECISION_NANO  2
 
 #define TSDB_TIME_PRECISION_MILLI_STR "ms"
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
@@ -286,9 +287,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_COMMIT_TIME            40960
 #define TSDB_DEFAULT_COMMIT_TIME        3600
 
-#define TSDB_MIN_PRECISION              TSDB_PRECISION_MILLI
-#define TSDB_MAX_PRECISION              TSDB_PRECISION_NANO
-#define TSDB_DEFAULT_PRECISION          TSDB_PRECISION_MILLI
+#define TSDB_MIN_PRECISION              TSDB_TIME_PRECISION_MILLI
+#define TSDB_MAX_PRECISION              TSDB_TIME_PRECISION_NANO
+#define TSDB_DEFAULT_PRECISION          TSDB_TIME_PRECISION_MILLI
 
 #define TSDB_MIN_COMP_LEVEL             0
 #define TSDB_MAX_COMP_LEVEL             2
@@ -357,12 +358,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TAOS_QTYPE_WAL      2 
 #define TAOS_QTYPE_CQ       3
 
-typedef enum {
-  TSDB_PRECISION_MILLI,
-  TSDB_PRECISION_MICRO,
-  TSDB_PRECISION_NANO
-} EPrecisionType;
-
 typedef enum {
   TSDB_SUPER_TABLE        = 0,  // super table
   TSDB_CHILD_TABLE        = 1,  // table created from super table

src/inc/tsdb.h

@@ -64,13 +64,14 @@ typedef struct {
   int8_t  compression;
 } STsdbCfg;
 
+typedef void TsdbRepoT;  // use void to hide implementation details from outside
+
 void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
 STsdbCfg *tsdbCreateDefaultCfg();
 void      tsdbFreeCfg(STsdbCfg *pCfg);
+STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-typedef void TsdbRepoT;  // use void to hide implementation details from outside
-
 int        tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
 int32_t    tsdbDropRepo(TsdbRepoT *repo);
 TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);

src/plugins/mqtt/src/mqttSystem.c

@@ -50,7 +50,7 @@ int32_t mqttInitSystem() {
   recntStatus.password = strstr(url, "@") != NULL ? strbetween(strstr(url, recntStatus.user_name), ":", "@") : NULL;
 
   if (strlen(url) == 0) {
-    mqttError("failed to initialize mqtt module, reason: url is null");
+    mqttTrace("mqtt module not init, url is null");
     return rc;
   }
 

src/query/inc/qExecutor.h

@@ -136,7 +136,7 @@ typedef struct SQuery {
   int64_t          intervalTime;
   int64_t          slidingTime;      // sliding time for sliding window query
   char             slidingTimeUnit;  // interval data type, used for daytime revise
-  int8_t           precision;
+  int16_t          precision;
   int16_t          numOfOutput;
   int16_t          fillType;
   int16_t          checkBuffer;  // check if the buffer is full during scan each block

src/query/inc/qpercentile.h

@@ -49,7 +49,7 @@ typedef struct tMemBucket {
   int32_t nTotalBufferSize;
   int32_t maxElemsCapacity;
   
-  int16_t pageSize;
+  int32_t pageSize;
   int16_t numOfTotalPages;
   int16_t numOfAvailPages; /* remain available buffer pages */
   

src/query/src/qExecutor.c

@@ -3964,10 +3964,12 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
 }
 
 int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-  int32_t code = TSDB_CODE_SUCCESS;
+  pQuery->precision = tsdbGetCfg(tsdb)->precision;
 
   setScanLimitationByResultBuffer(pQuery);
   changeExecuteScanOrder(pQuery, false);
@@ -5491,7 +5493,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   
   int tableIndex = 0;
   STimeWindow window = pQueryMsg->window;
-  taosArraySort( pTableIdList, compareTableIdInfo );
+  taosArraySort(pTableIdList, compareTableIdInfo);
   for(int32_t i = 0; i < numOfGroups; ++i) {
     SArray* pa = taosArrayGetP(groupInfo->pGroupList, i);
     size_t s = taosArrayGetSize(pa);
@@ -5961,6 +5963,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->useconds = 0;
   }
   
+  (*pRsp)->precision = htons(pQuery->precision);
   if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) {
     code = doDumpQueryResult(pQInfo, (*pRsp)->data);
   } else {

src/rpc/src/rpcMain.c

@@ -680,13 +680,13 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
   } 
 
   if (pConn) {
+    pConn->tretry = 0;
     pConn->ahandle = pContext->ahandle;
     sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle);
   } else {
     tError("%s %p, failed to set up connection(%s)", pRpc->label, pContext->ahandle, tstrerror(terrno));
   }
 
-  pConn->tretry = 0;
   return pConn;
 }
 

src/tsdb/src/tsdbMain.c

@@ -9,8 +9,7 @@
 #include "ttime.h"
 #include <sys/stat.h>
 
-#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI  // default precision
-#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
+#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
 #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
 #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
 #define TSDB_MIN_ID 0
@@ -79,6 +78,11 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
   if (pCfg != NULL) free(pCfg);
 }
 
+STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo) {
+  assert(repo != NULL);
+  return &((STsdbRepo*)repo)->config;
+}
+
 /**
  * Create a new TSDB repository
  * @param rootDir the TSDB repository root directory

src/tsdb/src/tsdbRead.c

@@ -364,12 +364,13 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   return true;
 }
 
-static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile) {
+static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precision) {
+  assert(precision >= TSDB_TIME_PRECISION_MICRO || precision <= TSDB_TIME_PRECISION_NANO);
   if (key == TSKEY_INITIAL_VAL) {
     return INT32_MIN;
   }
   
-  int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[0]));  // set the starting fileId
+  int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
     fid = INT32_MIN;
   }
@@ -1297,7 +1298,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
   // find the start data block in file
   if (!pQueryHandle->locateStart) {
     pQueryHandle->locateStart = true;
-    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pQueryHandle->pTsdb->config.daysPerFile);
+    STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
+    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
     
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);

src/util/src/ttimer.c

@@ -342,7 +342,6 @@ static void taosTimerLoopFunc(int signo) {
   int64_t now = taosGetTimestampMs();
 
   for (int i = 0; i < tListLen(wheels); i++) {
-    tmrTrace("begin processing wheel %d", i);
     // `expried` is a temporary expire list.
     // expired timers are first add to this list, then move
     // to expired queue as a batch to improve performance.
@@ -390,7 +389,6 @@ static void taosTimerLoopFunc(int signo) {
     }
 
     addToExpired(expired);
-    tmrTrace("end processing wheel %d", i);
   }
 }
 

tests/script/general/http/restful_full.sim

@@ -147,7 +147,7 @@ print ===============  step3 - db
 
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d ' select * from d1.t1;' 127.0.0.1:6020/rest/sql
 print 21-> $system_content
-if $system_content != @{"status":"error","code":1000,"desc":"invalid table name"}@ then
+if $system_content != @{"status":"error","code":1000,"desc":"invalid table id"}@ then
   return -1
 endi
 

tests/script/jenkins/basic.txt

@@ -59,7 +59,6 @@ cd ../../../debug; make
 ./test.sh -f general/db/delete_writing1.sim
 ./test.sh -f general/db/delete_writing2.sim
 ./test.sh -f general/db/delete.sim
-#not test ./test.sh -f general/db/dropdnodes.sim
 ./test.sh -f general/db/len.sim
 ./test.sh -f general/db/repeat.sim
 ./test.sh -f general/db/tables.sim
@@ -121,7 +120,7 @@ cd ../../../debug; make
 # ./test.sh -f general/parser/import_file.sim
 ./test.sh -f general/parser/lastrow.sim
 ./test.sh -f general/parser/nchar.sim
-./test.sh -f general/parser/null_char.sim
+#unsupport ./test.sh -f general/parser/null_char.sim
 ./test.sh -f general/parser/single_row_in_tb.sim
 ./test.sh -f general/parser/select_from_cache_disk.sim
 ./test.sh -f general/parser/limit.sim
@@ -214,7 +213,6 @@ cd ../../../debug; make
 ./test.sh -f general/tag/tinyint.sim
 
 ./test.sh -f general/user/authority.sim
-./test.sh -f general/user/basic1.sim
 ./test.sh -f general/user/monitor.sim
 ./test.sh -f general/user/pass_alter.sim
 ./test.sh -f general/user/pass_len.sim

tests/script/jenkins/general.txt

+cd ../../debug; cmake ..
+cd ../../debug; make
+
+./test.sh -f general/db/dropdnodes.sim
+./test.sh -f general/user/basic1.sim