[TD-10872]<feature>taosdemo support unicode chinese (#8796)

* [TD-10872]<feature>taosdemo support unicode chinese

* add to insert output

* add command line -C

* add --help

src/kit/taosdemo/inc/demo.h

@@ -158,6 +158,7 @@ extern char configDir[];
 #define DEFAULT_TOTAL_INSERT 0
 #define DEFAULT_TOTAL_AFFECT 0
 #define DEFAULT_DEMO_MODE true
+#define DEFAULT_CHINESE_OPT false
 #define DEFAULT_CREATE_BATCH 10
 #define DEFAULT_SUB_INTERVAL 10000
 #define DEFAULT_QUERY_INTERVAL 10000
@@ -329,6 +330,7 @@ typedef struct SArguments_S {
     uint64_t totalInsertRows;
     uint64_t totalAffectedRows;
     bool     demo_mode;  // use default column name and semi-random data
+    bool     chinese;
 } SArguments;
 
 typedef struct SColumn_S {

src/kit/taosdemo/src/demoCommandOpt.c

@@ -972,6 +972,10 @@ int parse_args(int argc, char *argv[]) {
                    (0 == strncmp(argv[i], "--escape-character",
                                  strlen("--escape-character")))) {
             g_args.escapeChar = true;
+        } else if ((0 == strncmp(argv[i], "-C", strlen("-C"))) ||
+        (0 == strncmp(argv[i], "--chinese",
+                      strlen("--chinese")))) {
+            g_args.chinese = true;
         } else if ((strcmp(argv[i], "-N") == 0) ||
                    (0 == strcmp(argv[i], "--normal-table"))) {
             g_args.demo_mode = false;

src/kit/taosdemo/src/demoData.c

@@ -238,16 +238,81 @@ float UNUSED_FUNC demo_phase_float() {
                    360);
 }
 
+static int usc2utf8(char* p, int unic) {
+    if ( unic <= 0x0000007F )
+    {
+        *p     = (unic & 0x7F);
+        return 1;
+    }
+    else if ( unic >= 0x00000080 && unic <= 0x000007FF )
+    {
+        *(p+1) = (unic & 0x3F) | 0x80;
+        *p     = ((unic >> 6) & 0x1F) | 0xC0;
+        return 2;
+    }
+    else if ( unic >= 0x00000800 && unic <= 0x0000FFFF )
+    {
+        *(p+2) = (unic & 0x3F) | 0x80;
+        *(p+1) = ((unic >>  6) & 0x3F) | 0x80;
+        *p     = ((unic >> 12) & 0x0F) | 0xE0;
+        return 3;
+    }
+    else if ( unic >= 0x00010000 && unic <= 0x001FFFFF )
+    {
+        *(p+3) = (unic & 0x3F) | 0x80;
+        *(p+2) = ((unic >>  6) & 0x3F) | 0x80;
+        *(p+1) = ((unic >> 12) & 0x3F) | 0x80;
+        *p     = ((unic >> 18) & 0x07) | 0xF0;
+        return 4;
+    }
+    else if ( unic >= 0x00200000 && unic <= 0x03FFFFFF )
+    {
+        *(p+4) = (unic & 0x3F) | 0x80;
+        *(p+3) = ((unic >>  6) & 0x3F) | 0x80;
+        *(p+2) = ((unic >> 12) & 0x3F) | 0x80;
+        *(p+1) = ((unic >> 18) & 0x3F) | 0x80;
+        *p     = ((unic >> 24) & 0x03) | 0xF8;
+        return 5;
+    }
+    else if ( unic >= 0x04000000 && unic <= 0x7FFFFFFF )
+    {
+        *(p+5) = (unic & 0x3F) | 0x80;
+        *(p+4) = ((unic >>  6) & 0x3F) | 0x80;
+        *(p+3) = ((unic >> 12) & 0x3F) | 0x80;
+        *(p+2) = ((unic >> 18) & 0x3F) | 0x80;
+        *(p+1) = ((unic >> 24) & 0x3F) | 0x80;
+        *p     = ((unic >> 30) & 0x01) | 0xFC;
+        return 6;
+    }
+    return 0;
+}
+
 void rand_string(char *str, int size) {
-    str[0] = 0;
-    if (size > 0) {
-        //--size;
-        int n;
-        for (n = 0; n < size; n++) {
-            int key = abs(taosRandom()) % (int)(sizeof(charset) - 1);
-            str[n] = charset[key];
+    if (g_args.chinese) {
+        char* pstr = str;
+        int move = 0;
+        while (size > 0) {
+            // Chinese Character need 3 bytes space
+            if (size < 3) {
+                break;
+            }
+            // Basic Chinese Character's Unicode is from 0x4e00 to 0x9fa5
+            int unic = 0x4e00 + rand() % (0x9fa5 - 0x4e00);
+            move = usc2utf8(pstr, unic);
+            pstr += move;
+            size -= move;
+        }
+    } else {
+        str[0] = 0;
+        if (size > 0) {
+            //--size;
+            int n;
+            for (n = 0; n < size; n++) {
+                int key = abs(taosRandom()) % (int)(sizeof(charset) - 1);
+                str[n] = charset[key];
+            }
+            str[n] = 0;
         }
-        str[n] = 0;
     }
 }
 

src/kit/taosdemo/src/demoInsert.c

@@ -1138,7 +1138,17 @@ void postFreeResource() {
     tmfree(g_randfloat_buff);
     tmfree(g_rand_current_buff);
     tmfree(g_rand_phase_buff);
-
+    tmfree(g_randdouble_buff);
+    tmfree(g_randuint_buff);
+    tmfree(g_randutinyint_buff);
+    tmfree(g_randusmallint_buff);
+    tmfree(g_randubigint_buff);
+    tmfree(g_randint);
+    tmfree(g_randuint);
+    tmfree(g_randbigint);
+    tmfree(g_randubigint);
+    tmfree(g_randfloat);
+    tmfree(g_randdouble);
     tmfree(g_sampleDataBuf);
 
     for (int l = 0; l < g_args.columnCount; l++) {

src/kit/taosdemo/src/demoJsonOpt.c

@@ -450,6 +450,25 @@ int getMetaFromInsertJsonFile(cJSON *root) {
         goto PARSE_OVER;
     }
 
+    cJSON *chineseOpt = cJSON_GetObjectItem(root, "chinese");  // yes, no,
+        if (chineseOpt && chineseOpt->type == cJSON_String &&
+        chineseOpt->valuestring != NULL) {
+            if (0 == strncasecmp(chineseOpt->valuestring, "yes", 3)) {
+                g_args.chinese = true;
+            } else if (0 == strncasecmp(chineseOpt->valuestring, "no", 2)) {
+                g_args.chinese = false;
+            } else {
+                g_args.chinese = DEFAULT_CHINESE_OPT;
+            }
+        } else if (!chineseOpt) {
+            g_args.chinese = DEFAULT_CHINESE_OPT;
+        } else {
+            errorPrint(
+                    "%s",
+                    "failed to read json, chinese input mistake\n");
+            goto PARSE_OVER;
+        }
+
     cJSON *answerPrompt =
         cJSON_GetObjectItem(root, "confirm_parameter_prompt");  // yes, no,
     if (answerPrompt && answerPrompt->type == cJSON_String &&

src/kit/taosdemo/src/demoMain.c

@@ -66,6 +66,7 @@ SArguments g_args = {
     DEFAULT_TOTAL_INSERT,      // totalInsertRows;
     DEFAULT_TOTAL_AFFECT,      // totalAffectedRows;
     DEFAULT_DEMO_MODE,         // demo_mode;
+    DEFAULT_CHINESE_OPT        // chinese
 };
 
 int main(int argc, char *argv[]) {

src/kit/taosdemo/src/demoOutput.c

@@ -313,6 +313,8 @@ void printHelp() {
            "Table prefix name. By default use 'd'.");
     printf("%s%s%s%s\n", indent, "-E, --escape-character", "\t",
            "Use escape character for Both Stable and normmal table name");
+    printf("%s%s%s%s\n", indent, "-C, --chinese", "\t",
+           "Use chinese characters as the data source for binary/nchar data");
     printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t",
            "The select sql file.");
     printf("%s%s%s%s\n", indent, "-N, --normal-table", "\t\t",
@@ -414,6 +416,8 @@ void printfInsertMeta() {
     printf("number of records per req:  \033[33m%u\033[0m\n", g_args.reqPerReq);
     printf("max sql length:             \033[33m%" PRIu64 "\033[0m\n",
            g_args.max_sql_len);
+    printf("random prepare data:        \033[33m%" PRId64 "\033[0m\n", g_args.prepared_rand);
+    printf("chinese:                    \033[33m%s\033[0m\n", g_args.chinese?"yes":"no");
 
     printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);