Merge branch 'develop' into feature/2.0tsdb

src/client/src/tscParseInsert.c

@@ -791,12 +791,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     sToken = tStrGetToken(sql, &index, false, 0, NULL);
     sql += index;
 
-    STagData *pTag = (STagData *)pCmd->payload;
+    tscAllocPayload(pCmd, sizeof(STagData));
+    STagData *pTag = (STagData *) pCmd->payload;
+
     memset(pTag, 0, sizeof(STagData));
     
-    /*
-     * the source super table is moved to the secondary position of the pTableMetaInfo list
-     */
+    //the source super table is moved to the secondary position of the pTableMetaInfo list
     if (pQueryInfo->numOfTables < 2) {
       tscAddEmptyMetaInfo(pQueryInfo);
     }
@@ -897,9 +897,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       index = 0;
       sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes);
       sql += index;
-      if (sToken.n == 0) {
-        break;
-      } else if (sToken.type == TK_RP) {
+
+      if (sToken.n == 0 || sToken.type == TK_RP) {
         break;
       }
 
@@ -913,11 +912,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
-
-      if ((pTagSchema[colIndex].type == TSDB_DATA_TYPE_BINARY || pTagSchema[colIndex].type == TSDB_DATA_TYPE_NCHAR) &&
-          sToken.n > pTagSchema[colIndex].bytes) {
-        return tscInvalidSQLErrMsg(pCmd->payload, "string too long", sToken.z);
-      }
     }
 
     index = 0;
@@ -1042,9 +1036,9 @@ int tsParseInsertSql(SSqlObj *pSql) {
   }
 
   // TODO: 2048 is added because TSDB_MAX_TAGS_LEN now is 65536, but TSDB_PAYLOAD_SIZE is 65380
-  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
-    return code;
-  }
+//  if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE + 2048)) != TSDB_CODE_SUCCESS) {
+//    return code;
+//  }
 
   if (NULL == pCmd->pTableList) {
     pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);

src/client/src/tscUtil.c

@@ -608,6 +608,9 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
 int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
   SSqlCmd* pCmd = &pSql->cmd;
 
+  // the expanded size when a row data is converted to SDataRow format
+  const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
+
   void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
   SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
 
@@ -627,7 +630,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
       return ret;
     }
 
-    int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2;
+    SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
+    int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE;
+
     if (dataBuf->nAllocSize < destSize) {
       while (dataBuf->nAllocSize < destSize) {
         dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5;
@@ -648,26 +653,30 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
       }
     }
 
-    SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
     tscSortRemoveDataBlockDupRows(pOneTableBlock);
-
     char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
     
     tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId,
         pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
 
-    int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2);
-    
+
+    int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE);
+
     pBlocks->tid = htonl(pBlocks->tid);
     pBlocks->uid = htobe64(pBlocks->uid);
     pBlocks->sversion = htonl(pBlocks->sversion);
     pBlocks->numOfRows = htons(pBlocks->numOfRows);
-    
-    pBlocks->len = htonl(len);
-    
+
     // erase the empty space reserved for binary data
-    len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
-    dataBuf->size += (len + sizeof(SSubmitBlk));
+    int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
+    assert(finalLen <= len);
+
+    dataBuf->size += (finalLen + sizeof(SSubmitBlk));
+    assert(dataBuf->size <= dataBuf->nAllocSize);
+
+    // the length does not include the SSubmitBlk structure
+    pBlocks->len = htonl(finalLen);
+
     dataBuf->numOfTables += 1;
   }
 

src/kit/taosdump/taosdump.c

@@ -644,14 +644,15 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) {
 
   (void)lseek(fd, 0, SEEK_SET);
 
+  STableRecord tableInfo;
   while (1) {
-    memset(&tableRecord, 0, sizeof(STableRecord));
-    ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord));
+    memset(&tableInfo, 0, sizeof(STableRecord));
+    ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
     if (ret <= 0) break;
     
-    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
-    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
-    taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
+    tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
+    tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;
+    taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
   }
 
   close(fd);
@@ -910,14 +911,22 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) {
 
   (void)lseek(fd, 0, SEEK_SET);
 
+  STableRecord tableInfo;
+  char tableName[TSDB_TABLE_NAME_LEN] ;
+  char metricName[TSDB_TABLE_NAME_LEN];
   while (1) {    
-    memset(&tableRecord, 0, sizeof(STableRecord));
-    ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord));
+    memset(&tableInfo, 0, sizeof(STableRecord));      
+    memset(tableName, 0, TSDB_TABLE_NAME_LEN);    
+    memset(metricName, 0, TSDB_TABLE_NAME_LEN);
+    ssize_t ret = read(fd, &tableInfo, sizeof(STableRecord));
     if (ret <= 0) break;
     
-    tableRecord.name[sizeof(tableRecord.name) - 1] = 0;
-    tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0;
-    taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp);
+    //tableInfo.name[sizeof(tableInfo.name) - 1] = 0;
+    //tableInfo.metric[sizeof(tableInfo.metric) - 1] = 0;    
+    //taosDumpTable(tableInfo.name, tableInfo.metric, arguments, fp);
+    tstrncpy(tableName, tableInfo.name, TSDB_TABLE_NAME_LEN-1);
+    tstrncpy(metricName, tableInfo.metric, TSDB_TABLE_NAME_LEN-1);
+    taosDumpTable(tableName, metricName, arguments, fp);
   }
 
   close(fd);

src/util/src/tconfig.c

@@ -111,21 +111,18 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
         wordfree(&full_path);
         return;
       }
+      
       if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
         strcpy(option, full_path.we_wordv[0]);
       }
+      
       wordfree(&full_path);
 
-      struct stat dirstat;
-      if (stat(option, &dirstat) < 0) {
-        int code = mkdir(option, 0755);
-        if (code < 0) {
-          uError("config option:%s, input value:%s, directory not exist, create fail with return code:%d",
-               cfg->option, input_value, code); 
-        } else {
-          uPrint("config option:%s, input value:%s, directory not exist, create with return code:%d",
-               cfg->option, input_value, code); 
-        }
+      int code = tmkdir(option, 0755);
+      if (code != 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        uError("config option:%s, input value:%s, directory not exist, create fail:%s",
+             cfg->option, input_value, strerror(errno)); 
       }
       cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
     } else {

src/util/src/tlog.c

@@ -276,14 +276,15 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
     }
   }
 
-  sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag);
+  char fileName[LOG_FILE_NAME_LEN + 50] = "\0";
+  sprintf(fileName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
   pthread_mutex_init(&tsLogObj.logMutex, NULL);
 
   umask(0);
-  tsLogObj.logHandle->fd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+  tsLogObj.logHandle->fd = open(fileName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
   if (tsLogObj.logHandle->fd < 0) {
-    printf("\nfailed to open log file:%s, reason:%s\n", name, strerror(errno));
+    printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
   taosLockFile(tsLogObj.logHandle->fd);
@@ -291,7 +292,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   // only an estimate for number of lines
   struct stat filestat;
   if (fstat(tsLogObj.logHandle->fd, &filestat) < 0) {
-    printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno));
+    printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
   size = (int32_t)filestat.st_size;

src/util/src/ttime.c

@@ -56,7 +56,7 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
 		year -= 1;
 	}
 
-  int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
+  int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + (int64_t)(367*mon)/12 + day) +
 		  year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
 
 	return (res + timezone);

tests/examples/c/demo.c

@@ -44,7 +44,9 @@ int main(int argc, char *argv[]) {
 
 
   taos_query(taos, "drop database demo");
-  if (taos_query(taos, "create database demo") != 0) {
+
+  result = taos_query(taos, "create database demo");
+  if (result == NULL) {
     printf("failed to create database, reason:%s\n", taos_errstr(taos));
     exit(1);
   }
@@ -53,7 +55,7 @@ int main(int argc, char *argv[]) {
   taos_query(taos, "use demo");
 
   // create table
-  if (taos_query(taos, "create table m1 (ts timestamp, speed int)") != 0) {
+  if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
     printf("failed to create table, reason:%s\n", taos_errstr(taos));
     exit(1);
   }
@@ -65,9 +67,10 @@ int main(int argc, char *argv[]) {
   // insert 10 records
   int i = 0;
   for (i = 0; i < 10; ++i) {
-    sprintf(qstr, "insert into m1 values (%ld, %d)", 1546300800000 + i * 1000, i * 10);
+    sprintf(qstr, "insert into m1 values (%ld, %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello");
+    printf("qstr: %s\n", qstr);
     if (taos_query(taos, qstr)) {
-      printf("failed to insert row: %i, reason:%s\n", i, taos_errstr(taos));
+      printf("insert row: %i, reason:%s\n", i, taos_errstr(taos));
     }
     //sleep(1);
   }
@@ -83,10 +86,11 @@ int main(int argc, char *argv[]) {
 
   TAOS_ROW    row;
   int         rows = 0;
-  int         num_fields = taos_field_count(taos);
+  int         num_fields = taos_field_count(result);
   TAOS_FIELD *fields = taos_fetch_fields(result);
-  char        temp[256];
+  char        temp[1024];
 
+  printf("num_fields = %d\n", num_fields);
   printf("select * from table, result:\n");
   // fetch the records row by row
   while ((row = taos_fetch_row(result))) {

tests/pytest/alter/alter_table_crash.py

@@ -47,7 +47,6 @@ class TDTestCase:
         tdSql.query("select * from st")
         tdSql.checkRows(1)
 
-
         print("==============Case 2: keep adding columns, restart taosd")
         tdSql.execute(
             "create table dt(ts timestamp, tbcol1 tinyint) tags(tgcol1 tinyint)")
@@ -72,7 +71,7 @@ class TDTestCase:
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
 
-        tdSql.query("select * from st")
+        tdSql.query("select * from dt")
         tdSql.checkRows(0)
 
     def stop(self):

tests/pytest/regressiontest.sh

@@ -141,5 +141,7 @@ python3 ./test.py -f query/querySort.py
 python3 ./test.py -f stream/stream1.py
 python3 ./test.py -f stream/stream2.py
 
+#alter table
+python3 ./test.py -f alter/alter_table_crash.py
 
 

tests/script/unique/http/opentsdb.sim

@@ -75,7 +75,7 @@ endi
 
 system_content curl -u root:taosdata -d  '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put
 print $system_content
-if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101}}],"failed":1,"success":0,"affected_rows":0}@ then
+if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then
   return -1
 endi
 
@@ -125,7 +125,7 @@ endi
 
 system_content curl -u root:taosdata -d  '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put
 print $system_content
-if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782}}],"failed":1,"success":0,"affected_rows":0}@ then
+if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then
   return -1
 endi