Merge branch 'develop' of github.com:taosdata/TDengine into dev/chr

src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h

@@ -51,10 +51,10 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset
 
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector
- * Method:    getResultTimePrecision
- * Signature: (J)J
+ * Method:    getResultTimePrecisionImp
+ * Signature: (JJ)I
  */
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecision
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecisionImp
         (JNIEnv *, jobject, jlong, jlong);
 
 /*

src/client/src/TSDBJNIConnector.c

@@ -113,7 +113,7 @@ static void jniGetGlobalMethod(JNIEnv *env) {
   g_rowdataSetFloatFp = (*env)->GetMethodID(env, g_rowdataClass, "setFloat", "(IF)V");
   g_rowdataSetDoubleFp = (*env)->GetMethodID(env, g_rowdataClass, "setDouble", "(ID)V");
   g_rowdataSetStringFp = (*env)->GetMethodID(env, g_rowdataClass, "setString", "(ILjava/lang/String;)V");
-  g_rowdataSetTimestampFp = (*env)->GetMethodID(env, g_rowdataClass, "setTimestamp", "(IJ)V");
+  g_rowdataSetTimestampFp = (*env)->GetMethodID(env, g_rowdataClass, "setTimestamp", "(IJI)V");
   g_rowdataSetByteArrayFp = (*env)->GetMethodID(env, g_rowdataClass, "setByteArray", "(I[B)V");
   (*env)->DeleteLocalRef(env, rowdataClass);
 
@@ -519,9 +519,11 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
                                jniFromNCharToByteArray(env, (char *)row[i], length[i]));
         break;
       }
-      case TSDB_DATA_TYPE_TIMESTAMP:
-        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i]));
+      case TSDB_DATA_TYPE_TIMESTAMP: {
+        int precision = taos_result_precision(result);
+        (*env)->CallVoidMethod(env, rowobj, g_rowdataSetTimestampFp, i, (jlong) * ((int64_t *)row[i]), precision);
         break;
+      }
       default:
         break;
     }
@@ -672,7 +674,15 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset(J
   return (*env)->NewStringUTF(env, (const char *)tsCharset);
 }
 
-JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TDDBJNIConnector_getResultTimePrecision(JNIEnv *env, jobject jobj, jlong con,
+/**
+ * Get Result Time Precision
+ * @param env           vm
+ * @param jobj          the TSDBJNIConnector java object
+ * @param con           the c connection pointer
+ * @param res           the TAOS_RES object, i.e. the SSqlObject
+ * @return precision    0:ms 1:us 2:ns
+ */
+JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultTimePrecisionImp(JNIEnv *env, jobject jobj, jlong con,
                                                                                       jlong res) {
   TAOS *tscon = (TAOS *)con;
   if (tscon == NULL) {

src/client/src/tscSQLParser.c

@@ -772,6 +772,10 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       pCmd->active = pCmd->pQueryInfo;
       pCmd->command = pCmd->pQueryInfo->command;
 
+      if (pTableMetaInfo->pTableMeta != NULL) {
+        pSql->res.precision = tscGetTableInfo(pTableMetaInfo->pTableMeta).precision;
+      }
+
       return TSDB_CODE_SUCCESS;  // do not build query message here
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -216,6 +216,16 @@ public class TSDBJNIConnector {
 
     private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData);
 
+    /**
+     * Get Result Time Precision.
+     * @return 0: ms, 1: us, 2: ns
+    */
+    public int getResultTimePrecision(long sqlObj) {
+        return this.getResultTimePrecisionImp(this.taos, sqlObj);
+    }
+
+    private native int getResultTimePrecisionImp(long connection, long result);
+
     /**
      * Execute close operation from C to release connection pointer by JNI
      *

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java

@@ -414,26 +414,40 @@ public class TSDBResultSetRowData {
      * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api
      */
     public void setTimestampValue(int colIndex, long value) {
-        setTimestamp(colIndex - 1, value);
+        setTimestamp(colIndex - 1, value, 0);
     }
 
     /**
      * !!! this method is invoked by JNI method and the index start from 0 in C implementations
+     * @param precision 0 : ms, 1 : us, 2 : ns
      */
-    public void setTimestamp(int col, long ts) {
-        //TODO: this implementation contains logical error
-        // when precision is us the (long ts) is 16 digital number
-        // when precision is ms, the (long ts) is 13 digital number
-        // we need a JNI function like this:
-        //      public void setTimestamp(int col, long epochSecond, long nanoAdjustment)
-        if (ts < 1_0000_0000_0000_0L) {
-            data.set(col, new Timestamp(ts));
-        } else {
-            long epochSec = ts / 1000_000l;
-            long nanoAdjustment = ts % 1000_000l * 1000l;
-            Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment));
-            data.set(col, timestamp);
+    public void setTimestamp(int col, long ts, int precision) {
+        long milliseconds = 0;
+        int fracNanoseconds = 0;
+        switch (precision) {
+            case 0: {
+                milliseconds = ts;
+                fracNanoseconds = (int)(ts*1_000_000%1_000_000_000);
+                break;
+            }
+            case 1: {
+                milliseconds = ts/1_000;
+                fracNanoseconds = (int)(ts*1_000%1_000_000_000);
+                break;
+            }
+            case 2: {
+                milliseconds = ts/1_000_000;
+                fracNanoseconds = (int)(ts%1_000_000_000);
+                break;
+            }
+            default: {
+                throw new IllegalArgumentException("precision is not valid. precision: " + precision);
+            }
         }
+
+        Timestamp tsObj = new Timestamp(milliseconds);
+        tsObj.setNanos(fracNanoseconds);
+        data.set(col, tsObj);
     }
 
     public Timestamp getTimestamp(int col, int nativeType) {

src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java

 package com.taosdata.jdbc;
 
 import org.junit.Test;
+import static org.junit.Assert.*;
 
 import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.util.ArrayList;
 import java.util.List;
 
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.management.ThreadMXBean;
+
 public class TSDBJNIConnectorTest {
 
     private static TSDBResultSetRowData rowData;
@@ -14,17 +19,68 @@ public class TSDBJNIConnectorTest {
     @Test
     public void test() {
         try {
+
+            try {
+                //change sleepSeconds when debugging with attach to process to find PID
+                int sleepSeconds = -1;
+                if (sleepSeconds>0) {
+                    RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+                    String jvmName = runtimeBean.getName();
+                    long pid = Long.valueOf(jvmName.split("@")[0]);
+                    System.out.println("JVM PID  = " + pid);
+
+                    Thread.sleep(sleepSeconds*1000);
+                }
+            }
+            catch (Exception e) {
+                e.printStackTrace();
+            }
+
             // init
-            TSDBJNIConnector.init("/etc/taos/taos.cfg", null, null, null);
+            TSDBJNIConnector.init("/etc/taos", null, null, null);
+
             // connect
             TSDBJNIConnector connector = new TSDBJNIConnector();
-            connector.connect("127.0.0.1", 6030, "unsign_jni", "root", "taosdata");
+            connector.connect("127.0.0.1", 6030, null, "root", "taosdata");
+
+            // setup
+            String setupSqlStrs[] = {"create database if not exists d precision \"us\"",
+                                      "create table if not exists d.t(ts timestamp, f int)",
+                                      "create database if not exists d2",
+                                      "create table if not exists d2.t2(ts timestamp, f int)",
+                                      "insert into d.t values(now+100s, 100)",
+                                      "insert into d2.t2 values(now+200s, 200)"
+                                      };
+            for (String setupSqlStr : setupSqlStrs) {
+                long setupSql = connector.executeQuery(setupSqlStr);
+
+                assertEquals(0, connector.getResultTimePrecision(setupSql));
+                if (connector.isUpdateQuery(setupSql)) {
+                    connector.freeResultSet(setupSql);
+                }
+            }
+
+            {
+                long sqlObj1 = connector.executeQuery("select * from d2.t2");
+                assertEquals(0, connector.getResultTimePrecision(sqlObj1));
+                List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
+                int code = connector.getSchemaMetaData(sqlObj1, columnMetaDataList);
+                rowData = new TSDBResultSetRowData(columnMetaDataList.size());
+                assertTrue(next(connector, sqlObj1));
+                assertEquals(0, connector.getResultTimePrecision(sqlObj1));
+                connector.freeResultSet(sqlObj1);
+            }
+
             // executeQuery
-            long pSql = connector.executeQuery("select * from unsign_jni.us_table");
+            long pSql = connector.executeQuery("select * from d.t");
+
             if (connector.isUpdateQuery(pSql)) {
                 connector.freeResultSet(pSql);
                 throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY);
             }
+
+            assertEquals(1, connector.getResultTimePrecision(pSql));
+
             // get schema
             List<ColumnMetaData> columnMetaDataList = new ArrayList<>();
             int code = connector.getSchemaMetaData(pSql, columnMetaDataList);
@@ -37,6 +93,8 @@ public class TSDBJNIConnectorTest {
             if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) {
                 throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0);
             }
+
+            assertEquals(1, connector.getResultTimePrecision(pSql));
             int columnSize = columnMetaDataList.size();
             // print metadata
             for (int i = 0; i < columnSize; i++) {
@@ -45,9 +103,8 @@ public class TSDBJNIConnectorTest {
             rowData = new TSDBResultSetRowData(columnSize);
             // iterate resultSet
             for (int i = 0; next(connector, pSql); i++) {
-//                System.out.println("col[" + i + "] size: " + rowData.getColSize());
-//                rowData.getData().stream().forEach(col -> System.out.print(col + "\t"));
-//                System.out.println();
+                assertEquals(1, connector.getResultTimePrecision(pSql));
+                System.out.println();
             }
             // close resultSet
             code = connector.freeResultSet(pSql);
@@ -86,4 +143,4 @@ public class TSDBJNIConnectorTest {
         }
     }
 
-}
\ No newline at end of file
+}

src/connector/python/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.10",
+    version="2.0.11",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/taos/cinterface.py

@@ -14,12 +14,22 @@ def _convert_microsecond_to_datetime(micro):
     return datetime.datetime.fromtimestamp(micro / 1000000.0)
 
 
-def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _convert_nanosecond_to_datetime(nanosec):
+    return datetime.datetime.fromtimestamp(nanosec / 1000000000.0)
+
+
+def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C bool row to python row
     """
     _timestamp_converter = _convert_millisecond_to_datetime
-    if micro:
+    if precision == FieldType.C_TIMESTAMP_MILLI:
+        _timestamp_converter = _convert_millisecond_to_datetime
+    elif precision == FieldType.C_TIMESTAMP_MICRO:
         _timestamp_converter = _convert_microsecond_to_datetime
+    elif precision == FieldType.C_TIMESTAMP_NANO:
+        _timestamp_converter = _convert_nanosecond_to_datetime
+    else:
+        raise DatabaseError("Unknown precision returned from database")
 
     return [
         None if ele == FieldType.C_BIGINT_NULL else _timestamp_converter(ele) for ele in ctypes.cast(
@@ -28,7 +38,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
             :abs(num_of_rows)]]
 
 
-def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_bool_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C bool row to python row
     """
     return [
@@ -38,7 +48,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
             :abs(num_of_rows)]]
 
 
-def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C tinyint row to python row
     """
     return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(
@@ -49,7 +59,7 @@ def _crow_tinyint_unsigned_to_python(
         data,
         num_of_rows,
         nbytes=None,
-        micro=False):
+        precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C tinyint row to python row
     """
     return [
@@ -59,7 +69,7 @@ def _crow_tinyint_unsigned_to_python(
             :abs(num_of_rows)]]
 
 
-def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_smallint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C smallint row to python row
     """
     return [
@@ -70,7 +80,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
 
 
 def _crow_smallint_unsigned_to_python(
-        data, num_of_rows, nbytes=None, micro=False):
+        data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C smallint row to python row
     """
     return [
@@ -80,14 +90,14 @@ def _crow_smallint_unsigned_to_python(
             :abs(num_of_rows)]]
 
 
-def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_int_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C int row to python row
     """
     return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(
         data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]]
 
 
-def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C int row to python row
     """
     return [
@@ -97,7 +107,7 @@ def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False):
             :abs(num_of_rows)]]
 
 
-def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_bigint_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C bigint row to python row
     """
     return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(
@@ -108,7 +118,7 @@ def _crow_bigint_unsigned_to_python(
         data,
         num_of_rows,
         nbytes=None,
-        micro=False):
+        precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C bigint row to python row
     """
     return [
@@ -118,21 +128,21 @@ def _crow_bigint_unsigned_to_python(
             :abs(num_of_rows)]]
 
 
-def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_float_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C float row to python row
     """
     return [None if math.isnan(ele) else ele for ele in ctypes.cast(
         data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]]
 
 
-def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_double_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C double row to python row
     """
     return [None if math.isnan(ele) else ele for ele in ctypes.cast(
         data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]]
 
 
-def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_binary_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C binary row to python row
     """
     assert(nbytes is not None)
@@ -140,7 +150,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
         'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
 
-def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
+def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
@@ -159,7 +169,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     return res
 
 
-def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C binary row to python row
     """
     assert(nbytes is not None)
@@ -178,7 +188,7 @@ def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
     return res
 
 
-def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldType.C_TIMESTAMP_UNKNOWN):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
@@ -448,8 +458,7 @@ class CTaosInterface(object):
             result, ctypes.byref(pblock))
         if num_of_rows == 0:
             return None, 0
-        isMicro = (CTaosInterface.libtaos.taos_result_precision(
-            result) == FieldType.C_TIMESTAMP_MICRO)
+        precision = CTaosInterface.libtaos.taos_result_precision(result)
         blocks = [None] * len(fields)
         fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
         fieldLen = [
@@ -462,7 +471,7 @@ class CTaosInterface(object):
             if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
                 raise DatabaseError("Invalid data type returned from database")
             blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](
-                data, num_of_rows, fieldLen[i], isMicro)
+                data, num_of_rows, fieldLen[i], precision)
 
         return blocks, abs(num_of_rows)
 
@@ -472,8 +481,7 @@ class CTaosInterface(object):
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)
         if pblock:
             num_of_rows = 1
-            isMicro = (CTaosInterface.libtaos.taos_result_precision(
-                result) == FieldType.C_TIMESTAMP_MICRO)
+            precision = CTaosInterface.libtaos.taos_result_precision(result)
             blocks = [None] * len(fields)
             fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
             fieldLen = [
@@ -490,7 +498,7 @@ class CTaosInterface(object):
                     blocks[i] = [None]
                 else:
                     blocks[i] = _CONVERT_FUNC[fields[i]['type']](
-                        data, num_of_rows, fieldLen[i], isMicro)
+                        data, num_of_rows, fieldLen[i], precision)
         else:
             return None, 0
         return blocks, abs(num_of_rows)

src/connector/python/taos/constants.py

@@ -40,3 +40,5 @@ class FieldType(object):
     # Timestamp precision definition
     C_TIMESTAMP_MILLI = 0
     C_TIMESTAMP_MICRO = 1
+    C_TIMESTAMP_NANO = 2
+    C_TIMESTAMP_UNKNOWN = 3

src/mnode/src/mnodeTable.c

@@ -1475,6 +1475,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) {
   SSchema *schema = (SSchema *) (pStable->schema + col);
   ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
   schema->bytes = pAlter->schema[0].bytes;
+  pStable->sversion++;
   mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          name, schema->bytes);
 
@@ -1504,6 +1505,7 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) {
   SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns);
   ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
   schema->bytes = pAlter->schema[0].bytes;
+  pStable->tversion++;
   mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          name, schema->bytes);
 

src/query/src/qExecutor.c

@@ -7527,7 +7527,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type
   if (IS_VAR_DATA_TYPE(type)) {
     // Binary data overflows for sort of unknown reasons. Let trim the overflow data
     if (varDataTLen(val) > bytes) {
-      int32_t len = bytes - VARSTR_HEADER_SIZE;   // remain available space
+      int32_t maxLen = bytes - VARSTR_HEADER_SIZE;
+      int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val);
       memcpy(varDataVal(output), varDataVal(val), len);
       varDataSetLen(output, len);
     } else {

src/tsdb/src/tsdbMain.c

@@ -812,6 +812,7 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) {
       STable *pTable = pMeta->tables[i];
       if (pTable == NULL) continue;
       pTable->restoreColumnNum = 0;  
+      pTable->hasRestoreLastColumn = false;
     }
   }
 
@@ -895,6 +896,7 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) {
     maxTableIdx = i;
     if (cacheLastCol) {
       pTable->restoreColumnNum = 0;
+      pTable->hasRestoreLastColumn = false;
     } 
   }
 
@@ -913,6 +915,7 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) {
       }
       if (need_free_last_col) {
         tsdbFreeLastColumns(pTable);
+        pTable->hasRestoreLastColumn = false;
       }
     }    
   }

src/tsdb/src/tsdbMeta.c

@@ -148,7 +148,9 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
   return 0;
 
 _err:
-  tsdbFreeTable(super);
+  if (newSuper) {
+    tsdbFreeTable(super);
+  }
   tsdbFreeTable(table);
   return -1;
 }
@@ -211,7 +213,7 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_
   }
 
   char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
-  assert(type == pCol->type && bytes == pCol->bytes);
+  assert(type == pCol->type && bytes >= pCol->bytes);
 
   // if (val != NULL && IS_VAR_DATA_TYPE(type)) {
   //   assert(varDataLen(val) < pCol->bytes);
@@ -607,6 +609,7 @@ void tsdbFreeLastColumns(STable* pTable) {
   pTable->maxColNum = 0;
   pTable->lastColSVersion = -1;
   pTable->restoreColumnNum = 0;
+  pTable->hasRestoreLastColumn = false;
 }
 
 int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) {
@@ -643,6 +646,7 @@ int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema) {
   pTable->lastColSVersion = schemaVersion(pSchema);
   pTable->maxColNum = numOfColumn;
   pTable->restoreColumnNum = 0;
+  pTable->hasRestoreLastColumn = false;
   return 0;
 }
 
@@ -655,7 +659,7 @@ int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema) {
     return 0;
   }
   
-  tsdbInfo("tsdbUpdateLastColSchema:%s,%d->%d", pTable->name->data, pTable->lastColSVersion, schemaVersion(pNewSchema));
+  tsdbDebug("tsdbUpdateLastColSchema:%s,%d->%d", pTable->name->data, pTable->lastColSVersion, schemaVersion(pNewSchema));
   
   int16_t numOfCols = pNewSchema->numOfCols;
   SDataCol *lastCols = (SDataCol*)malloc(numOfCols * sizeof(SDataCol));
@@ -800,6 +804,7 @@ static STable *tsdbNewTable() {
   pTable->lastCols = NULL;
   pTable->restoreColumnNum = 0;
   pTable->maxColNum = 0;
+  pTable->hasRestoreLastColumn = false;
   pTable->lastColSVersion = -1;
   return pTable;
 }

src/tsdb/src/tsdbRead.c

@@ -2520,7 +2520,7 @@ static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) {
     int32_t numOfCols = pTable->maxColNum;
     
     if (pTable->lastCols == NULL || pTable->maxColNum <= 0) {
-      tsdbWarn("no last cached for table, uid:%" PRIu64 ",tid:%d", pTable->tableId.uid, pTable->tableId.tid);
+      tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid);
       continue;
     }
     

tests/pytest/fulltest.sh

@@ -25,6 +25,8 @@ python3 ./test.py -f insert/special_character_show.py
 python3 bug2265.py
 python3 ./test.py -f insert/bug3654.py
 python3 ./test.py -f insert/insertDynamicColBeforeVal.py
+python3 ./test.py -f insert/in_function.py
+python3 ./test.py -f insert/modify_column.py
 
 #table
 python3 ./test.py -f table/alter_wal0.py

tests/pytest/insert/modify_column.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import datetime
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        # test case for https://jira.taosdata.com:18080/browse/TD-4541
+
+        tdLog.info("=============== step1,check normal table")    
+
+        tdLog.info("=============== step1.1,drop table && create table")    
+        cmd1 = 'drop table if exists length11 ;'
+        cmd2 = 'create table length11 (ts timestamp,lengthbia binary(10),lengthnchar nchar(20));'
+        tdLog.info(cmd1)
+        tdSql.execute(cmd1)
+        tdLog.info(cmd2)
+        tdSql.execute(cmd2)
+
+        tdLog.info("=============== step1.2,insert table right data") 
+        cmd1 = 'insert into length11 values(now,\'aaaaaaaaaa\',\'bbbbbbbbbbbbbbbbbbbb\') ;'
+        tdLog.info(cmd1)
+        tdSql.execute(cmd1)         
+        tdSql.query('select * from length11 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+        tdLog.info("=============== step1.3,insert table wrong data") 
+        cmd1 = 'insert into length11 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("string data overflow")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("insert wrong data error catched")      
+        tdSql.query('select * from length11 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+        tdLog.info("=============== step1.4,modify columu length ") 
+        cmd1 = 'alter table length11 modify column lengthbia binary(10) ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one")   
+
+        cmd2 = 'alter table length11 modify column lengthnchar nchar(20);'
+        tdLog.info(cmd2)
+        tdSql.error(cmd2)
+        try:
+            tdSql.execute(cmd2)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one") 
+
+        cmd3 = 'alter table length11 modify column lengthbia binary(11) ;'
+        cmd4 = 'describe length11 ;'
+        tdLog.info(cmd3)
+        tdSql.execute(cmd3) 
+        tdLog.info(cmd4)
+        tdSql.execute(cmd4) 
+        tdSql.query('describe length11 ;')
+        tdSql.checkData(1,2,11) 
+
+        cmd5 = 'alter table length11 modify column lengthnchar nchar(21);'
+        cmd6 = 'describe length11 ;'
+        tdLog.info(cmd5)
+        tdSql.execute(cmd5) 
+        tdLog.info(cmd6)
+        tdSql.execute(cmd6) 
+        tdSql.query('describe length11 ;')
+        tdSql.checkData(2,2,21) 
+
+        tdSql.query('select * from length11 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+
+        tdLog.info("=============== step1.5,insert table right data") 
+        cmd1 = 'insert into length11 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd1)    
+        tdSql.execute(cmd1)  
+        tdSql.query('select * from length11 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+
+
+
+
+        tdLog.info("=============== step2,check stable table and tag")    
+
+        tdLog.info("=============== step2.1,drop table && create table")    
+        cmd1 = 'drop table if exists length1 ;'
+        cmd2 = 'drop table if exists length2 ;'
+        cmd3 = 'drop table if exists length2 ;'
+        cmd4 = 'drop table if exists lengthsta1 ;'
+        cmd5 = 'create stable lengthsta1(ts timestamp,lengthbia binary(10),lengthnchar nchar(20)) tags (tlengthbia binary(15),tlengthnchar nchar(25)) ;'
+        cmd6 = 'create table length1 using lengthsta1 tags(\'aaaaabbbbbaaaaa\',\'bbbbbaaaaabbbbbaaaaabbbbb\') ; '
+        tdLog.info(cmd1)
+        tdSql.execute(cmd1)
+        tdLog.info(cmd2)
+        tdSql.execute(cmd2)
+        tdLog.info(cmd3)
+        tdSql.execute(cmd3)
+        tdLog.info(cmd4)
+        tdSql.execute(cmd4)
+        tdLog.info(cmd5)
+        tdSql.execute(cmd5)
+        tdLog.info(cmd6)
+        tdSql.execute(cmd6)
+
+        tdLog.info("=============== step2.2,insert table right data") 
+        cmd1 = 'insert into length1 values(now,\'aaaaaaaaaa\',\'bbbbbbbbbbbbbbbbbbbb\') ;'
+        tdLog.info(cmd1)
+        tdSql.execute(cmd1)         
+        tdSql.query('select * from length1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+        tdLog.info("=============== step2.3,insert table wrong data") 
+        cmd1 = 'insert into length1 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("string data overflow")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("insert wrong data error catched")      
+        tdSql.query('select * from length1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+        tdLog.info("=============== step2.4,modify columu length ") 
+        cmd0 = 'alter table length1 modify column lengthbia binary(10) ;'
+        tdLog.info(cmd0)
+        tdSql.error(cmd0)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("invalid operation: column can only be modified by super table")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("invalid operation: column can only be modified by super table")  
+
+        cmd1 = 'alter table lengthsta1 modify column lengthbia binary(10) ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one")   
+
+        cmd2 = 'alter table lengthsta1 modify column lengthnchar nchar(20);'
+        tdLog.info(cmd2)
+        tdSql.error(cmd2)
+        try:
+            tdSql.execute(cmd2)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one") 
+
+        cmd3 = 'alter table lengthsta1 modify column lengthbia binary(11) ;'
+        cmd4 = 'describe lengthsta1 ;'
+        tdLog.info(cmd3)
+        tdSql.execute(cmd3) 
+        tdLog.info(cmd4)
+        tdSql.execute(cmd4) 
+        tdSql.query('describe length1 ;')
+        tdSql.checkData(1,2,11) 
+
+        cmd5 = 'alter table lengthsta1 modify column lengthnchar nchar(21);'
+        cmd6 = 'describe lengthsta1 ;'
+        tdLog.info(cmd5)
+        tdSql.execute(cmd5) 
+        tdLog.info(cmd6)
+        tdSql.execute(cmd6) 
+        tdSql.query('describe lengthsta1 ;')
+        tdSql.checkData(2,2,21) 
+
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb')
+
+
+        tdLog.info("=============== step2.5,insert table right data") 
+        cmd1 = 'insert into length1 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd1)    
+        tdSql.execute(cmd1)  
+        tdSql.query('select * from length1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+
+
+
+        tdLog.info("=============== step2.6,create table wrong tag") 
+        cmd1 = 'create table length2 using lengthsta1 tags(\'aaaaabbbbbaaaaa1\',\'bbbbbaaaaabbbbbaaaaabbbbb1\')  ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("invalid operation: tag value too long")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("invalid operation: tag value too long")      
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+
+        tdLog.info("=============== step2.7,modify tag columu length ")  
+        cmd1 = 'alter table lengthsta1 modify tag tlengthbia binary(15) ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one")   
+
+        cmd2 = 'alter table lengthsta1 modify tag tlengthnchar nchar(25);'
+        tdLog.info(cmd2)
+        tdSql.error(cmd2)
+        try:
+            tdSql.execute(cmd2)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one") 
+
+        cmd3 = 'alter table lengthsta1 modify tag tlengthbia binary(16) ;'
+        cmd4 = 'describe lengthsta1 ;'
+        tdLog.info(cmd3)
+        tdSql.execute(cmd3) 
+        tdLog.info(cmd4)
+        tdSql.execute(cmd4) 
+        tdSql.query('describe lengthsta1 ;')
+        tdSql.checkData(3,2,16) 
+
+        cmd5 = 'alter table lengthsta1 modify tag tlengthnchar nchar(26);'
+        cmd6 = 'describe lengthsta1 ;'
+        tdLog.info(cmd5)
+        tdSql.execute(cmd5) 
+        tdLog.info(cmd6)
+        tdSql.execute(cmd6) 
+        tdSql.query('describe lengthsta1 ;')
+        tdSql.checkData(4,2,26) 
+
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.checkData(0,3,'aaaaabbbbbaaaaa') 
+        tdSql.checkData(0,4,'bbbbbaaaaabbbbbaaaaabbbbb')        
+
+
+        tdLog.info("=============== step2.8,creat tag right data and insert data") 
+        cmd1 = 'create table length2 using lengthsta1 tags(\'aaaaabbbbbaaaaa1\',\'bbbbbaaaaabbbbbaaaaabbbbb1\')  ;'
+        tdLog.info(cmd1)    
+        tdSql.execute(cmd1)  
+        tdSql.query('describe length2 ;')
+        tdSql.checkData(3,2,16) 
+        tdSql.checkData(4,2,26) 
+
+        cmd2 = 'insert into length2 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd2)    
+        tdSql.execute(cmd2)  
+        tdSql.query('select * from length2 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.checkData(0,3,'aaaaabbbbbaaaaa1') 
+        tdSql.checkData(0,4,'bbbbbaaaaabbbbbaaaaabbbbb1')  
+
+
+        tdLog.info("=============== step2.9,modify tag columu length again ")  
+        cmd1 = 'alter table lengthsta1 modify tag tlengthbia binary(16) ;'
+        tdLog.info(cmd1)
+        tdSql.error(cmd1)
+        try:
+            tdSql.execute(cmd1)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one")   
+
+        cmd2 = 'alter table lengthsta1 modify tag tlengthnchar nchar(26);'
+        tdLog.info(cmd2)
+        tdSql.error(cmd2)
+        try:
+            tdSql.execute(cmd2)
+            tdLog.exit("new column length should be bigger than old one")
+        except Exception as e:
+            tdLog.info(repr(e))
+            tdLog.info("new column length should be bigger than old one") 
+
+        cmd3 = 'alter table lengthsta1 modify tag tlengthbia binary(20) ;'
+        cmd4 = 'describe lengthsta1 ;'
+        tdLog.info(cmd3)
+        tdSql.execute(cmd3) 
+        tdLog.info(cmd4)
+        tdSql.execute(cmd4) 
+        tdSql.query('describe lengthsta1 ;')
+        tdSql.checkData(3,2,20) 
+
+        cmd5 = 'alter table lengthsta1 modify tag tlengthnchar nchar(30);'
+        cmd6 = 'describe lengthsta1 ;'
+        tdLog.info(cmd5)
+        tdSql.execute(cmd5) 
+        tdLog.info(cmd6)
+        tdSql.execute(cmd6) 
+        tdSql.query('describe lengthsta1 ;')
+        tdSql.checkData(4,2,30) 
+
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.checkData(0,3,'aaaaabbbbbaaaaa1') 
+        tdSql.checkData(0,4,'bbbbbaaaaabbbbbaaaaabbbbb1')        
+
+
+        tdLog.info("=============== step2.10,creat tag right data and insert data again") 
+        cmd1 = 'create table length3 using lengthsta1 tags(\'aaaaabbbbbaaaaabbbbb\',\'bbbbbaaaaabbbbbaaaaabbbbbaaaaa\')  ;'
+        tdLog.info(cmd1)    
+        tdSql.execute(cmd1)  
+        tdSql.query('describe length3 ;')
+        tdSql.checkData(3,2,20) 
+        tdSql.checkData(4,2,30) 
+
+        cmd2 = 'insert into length3 values(now,\'aaaaaaaaaa1\',\'bbbbbbbbbbbbbbbbbbbb1\') ;'
+        tdLog.info(cmd2)    
+        tdSql.execute(cmd2)  
+        tdSql.query('select * from length3 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.query('select * from lengthsta1 order by ts desc')
+        tdSql.checkData(0,1,'aaaaaaaaaa1') 
+        tdSql.checkData(0,2,'bbbbbbbbbbbbbbbbbbbb1')
+        tdSql.checkData(0,3,'aaaaabbbbbaaaaabbbbb') 
+        tdSql.checkData(0,4,'bbbbbaaaaabbbbbaaaaabbbbbaaaaa')  
+
+
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/script/general/http/grafana_bug.sim

@@ -20,17 +20,22 @@ sql use db
 sql create table tb (ts timestamp, val int, val1 int, val2 int)
 sql create table tb2 (ts timestamp, val int, val1 int, val2 int)
 sql create table t2 (ts timestamp, val int)
+sql create table tb3 (ts timestamp, val int, val1 int, val2 int)
 
 sql insert into tb values('2020-01-01 00:00:00.000', 1, 11, 21)
 sql insert into tb values('2020-01-02 00:00:00.000', 1, 12, 22)
 sql insert into tb values('2020-01-03 00:00:00.000', 2, 13, 23)
 sql insert into tb values('2020-01-04 00:00:00.000', 2, 14, 24)
 
+
 sql insert into tb2 values('2020-01-01 00:00:00.000', 21, 211, 221)
 sql insert into tb2 values('2020-01-02 00:00:00.000', 21, 212, 222)
 sql insert into tb2 values('2020-01-03 00:00:00.000', 22, 213, 223)
 sql insert into tb2 values('2020-01-04 00:00:00.000', 22, 214, 224)
 
+
+sql insert into tb3 values('2020-01-01 00:00:00.000', NULL, NULL, NULL)
+sql insert into tb3 values('2020-01-02 00:00:00.000', NULL, NULL, NULL)
 print ===============  step1 - one query, 1 column, with timestamp
 
 system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select ts from db.tb where ts >= 1577980800000 "} ]' 127.0.0.1:7111/grafana/query
@@ -235,4 +240,11 @@ if $system_content != @[{"refId":"B","target":"BB{val2:223,}","datapoints":[[213
   return -1
 endi
 
+print ===============  step26 - 2 column, no timestamp, NULL
+system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select * from db.tb3 "} ]' 127.0.0.1:7111/grafana/query
+print step1-> $system_content
+if $system_content != @[{"refId":"A","target":"{val1:nil, val2:nil}","datapoints":[[null,1577808000000],[null,1577894400000]]}]@ then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file