enh: column/row max length support up to 64K

docs/examples/c/async_query_example.c

@@ -8,7 +8,7 @@
 #include <sys/time.h>
 #include <taos.h>
 
-typedef int16_t VarDataLenT;
+typedef uint16_t VarDataLenT;
 
 #define TSDB_NCHAR_SIZE sizeof(int32_t)
 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

docs/examples/c/query_example.c

@@ -6,7 +6,7 @@
 #include <string.h>
 #include <taos.h>
 
-typedef int16_t VarDataLenT;
+typedef uint16_t VarDataLenT;
 
 #define TSDB_NCHAR_SIZE sizeof(int32_t)
 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT)

include/libs/function/function.h

@@ -99,11 +99,11 @@ typedef struct SSubsidiaryResInfo {
 } SSubsidiaryResInfo;
 
 typedef struct SResultDataInfo {
-  int16_t precision;
-  int16_t scale;
-  int16_t type;
-  int16_t bytes;
-  int32_t interBufSize;
+  int16_t  precision;
+  int16_t  scale;
+  int16_t  type;
+  uint16_t bytes;
+  int32_t  interBufSize;
 } SResultDataInfo;
 
 #define GET_RES_INFO(ctx)        ((ctx)->resultInfo)

include/libs/nodes/cmdnodes.h

@@ -30,7 +30,7 @@ extern "C" {
 
 #define SHOW_CREATE_DB_RESULT_COLS       2
 #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
-#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE)
+#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN)
 
 #define SHOW_CREATE_TB_RESULT_COLS       2
 #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE)

include/libs/parser/parser.h

@@ -114,7 +114,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
 
 int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
                     STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
-                    char* msgBuf, int16_t msgBufLen);
+                    char* msgBuf, int32_t msgBufLen);
 int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
 int     rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength);
 

include/util/tdef.h

@@ -236,8 +236,11 @@ typedef enum ELogicConditionType {
  *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
  *  - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
  *    the final value is 65531-(4096-1)*4 = 49151.
+ *  
+ *  History value:49151/65531
+ *  - 65531 compatible with 2.0
  */
-#define TSDB_MAX_BYTES_PER_ROW 49151
+#define TSDB_MAX_BYTES_PER_ROW 65531
 #define TSDB_MAX_TAGS_LEN      16384
 #define TSDB_MAX_TAGS          128
 
@@ -406,9 +409,9 @@ typedef enum ELogicConditionType {
 #define TSDB_EXPLAIN_RESULT_ROW_SIZE    (16 * 1024)
 #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
 
-#define TSDB_MAX_FIELD_LEN             16384
-#define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384
-#define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384
+#define TSDB_MAX_FIELD_LEN             65519 // compatible with 2.0
+#define TSDB_MAX_BINARY_LEN            TSDB_MAX_FIELD_LEN  // 16384:65519
+#define TSDB_MAX_NCHAR_LEN             TSDB_MAX_FIELD_LEN  // 16384:65519
 #define PRIMARYKEY_TIMESTAMP_COL_ID    1
 #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))
 

source/client/src/clientSmlJson.c

@@ -575,7 +575,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
     uError("OTD:invalid type(%s) for JSON String", typeStr);
     return TSDB_CODE_TSC_INVALID_JSON_TYPE;
   }
-  pVal->length = (int16_t)strlen(value->valuestring);
+  pVal->length = strlen(value->valuestring);
 
   if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
     return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;

source/dnode/mnode/impl/src/mndFunc.c

@@ -475,7 +475,7 @@ RETRIEVE_FUNC_OVER:
   return code;
 }
 
-static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) {
+static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {
   char *msg = "unknown";
   if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) {
     return msg;

source/libs/executor/src/timewindowoperator.c

@@ -1111,7 +1111,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
 
   bool    masterScan = true;
   int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
-  int16_t bytes = pStateColInfoData->info.bytes;
+  int32_t bytes = pStateColInfoData->info.bytes;
 
   SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
   TSKEY*           tsList = (TSKEY*)pColInfoData->pData;

source/libs/function/inc/tpercentile.h

@@ -53,7 +53,7 @@ typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *va
 typedef struct tMemBucket {
   int16_t            numOfSlots;
   int16_t            type;
-  int16_t            bytes;
+  int32_t            bytes;
   int32_t            total;
   int32_t            elemPerPage;  // number of elements for each object
   int32_t            maxCapacity;  // maximum allowed number of elements that can be sort directly to get the result
@@ -67,7 +67,7 @@ typedef struct tMemBucket {
   SHashObj          *groupPagesMap;  // disk page map for different groups;
 } tMemBucket;
 
-tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
+tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval);
 
 void tMemBucketDestroy(tMemBucket *pBucket);
 

source/libs/function/src/builtinsimpl.c

@@ -196,11 +196,11 @@ typedef struct SMavgInfo {
 } SMavgInfo;
 
 typedef struct SSampleInfo {
-  int32_t samples;
-  int32_t totalPoints;
-  int32_t numSampled;
-  uint8_t colType;
-  int16_t colBytes;
+  int32_t  samples;
+  int32_t  totalPoints;
+  int32_t  numSampled;
+  uint8_t  colType;
+  uint16_t colBytes;
 
   STuplePos nullTuplePos;
   bool      nullTupleSaved;
@@ -220,7 +220,7 @@ typedef struct STailInfo {
   int32_t     numAdded;
   int32_t     offset;
   uint8_t     colType;
-  int16_t     colBytes;
+  uint16_t    colBytes;
   STailItem** pItems;
 } STailInfo;
 
@@ -233,7 +233,7 @@ typedef struct SUniqueItem {
 typedef struct SUniqueInfo {
   int32_t   numOfPoints;
   uint8_t   colType;
-  int16_t   colBytes;
+  uint16_t  colBytes;
   bool      hasNull;  // null is not hashable, handle separately
   SHashObj* pHash;
   char      pItems[];
@@ -247,13 +247,13 @@ typedef struct SModeItem {
 
 typedef struct SModeInfo {
   uint8_t   colType;
-  int16_t   colBytes;
+  uint16_t  colBytes;
   SHashObj* pHash;
 
   STuplePos nullTuplePos;
   bool      nullTupleSaved;
 
-  char*     buf; // serialize data buffer
+  char* buf;  // serialize data buffer
 } SModeInfo;
 
 typedef struct SDerivInfo {

source/libs/function/src/tpercentile.c

@@ -236,7 +236,7 @@ static void resetSlotInfo(tMemBucket *pBucket) {
   }
 }
 
-tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
+tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval) {
   tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
   if (pBucket == NULL) {
     return NULL;

source/libs/parser/src/parInsertSml.c

@@ -242,7 +242,7 @@ end:
 
 int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
                     STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
-                    char* msgBuf, int16_t msgBufLen) {
+                    char* msgBuf, int32_t msgBufLen) {
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
 
   SSchema*       pTagsSchema = getTableTagSchema(pTableMeta);

source/libs/scalar/src/sclfunc.c

@@ -12,7 +12,7 @@ typedef double (*_double_fn)(double);
 typedef double (*_double_fn_2)(double, double);
 typedef int (*_conv_fn)(int);
 typedef void (*_trim_fn)(char *, char *, int32_t, int32_t);
-typedef int16_t (*_len_fn)(char *, int32_t);
+typedef uint16_t (*_len_fn)(char *, int32_t);
 
 /** Math functions **/
 static double tlog(double v) { return log(v); }
@@ -286,9 +286,9 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
 }
 
 /** String functions **/
-static int16_t tlength(char *input, int32_t type) { return varDataLen(input); }
+static VarDataLenT tlength(char *input, int32_t type) { return varDataLen(input); }
 
-static int16_t tcharlength(char *input, int32_t type) {
+static VarDataLenT tcharlength(char *input, int32_t type) {
   if (type == TSDB_DATA_TYPE_VARCHAR) {
     return varDataLen(input);
   } else {  // NCHAR
@@ -377,7 +377,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) {
+static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) {
   if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
     TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
     int32_t len = varDataLen(input);
@@ -457,7 +457,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
       continue;
     }
 
-    int16_t dataLen = 0;
+    VarDataLenT dataLen = 0;
     for (int32_t i = 0; i < inputNum; ++i) {
       int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
       input[i] = colDataGetData(pInputData[i], rowIdx);
@@ -526,8 +526,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
       continue;
     }
 
-    int16_t dataLen = 0;
-    bool    hasNull = false;
+    VarDataLenT dataLen = 0;
+    bool        hasNull = false;
     for (int32_t i = 1; i < inputNum; ++i) {
       if (colDataIsNull_s(pInputData[i], k) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) {
         hasNull = true;
@@ -695,7 +695,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 /** Conversion functions **/
 int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int16_t inputType = GET_PARAM_TYPE(&pInput[0]);
-  int16_t inputLen = GET_PARAM_BYTES(&pInput[0]);
+  int32_t inputLen = GET_PARAM_BYTES(&pInput[0]);
   int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
   int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]);
 

source/util/src/tcompare.c

@@ -1241,7 +1241,7 @@ int32_t taosArrayCompareString(const void *a, const void *b) {
 int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) {
   SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER;
 
-  ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
+  ASSERT(varDataTLen(pRight) <= TSDB_MAX_FIELD_LEN);
   size_t pLen = varDataLen(pRight);
   size_t sz = varDataLen(pLeft);
 

tools/shell/src/shellEngine.c

@@ -695,7 +695,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
 
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_JSON: {
-      int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
+      uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
       if (bytes > shell.args.displayWidth) {
         return TMAX(shell.args.displayWidth, width);
       } else {