Merge branch '3.0' of https://github.com/taosdata/TDengine into refact/tsdb_optimize

docs/en/10-deployment/01-deploy.md

@@ -114,7 +114,9 @@ The above process can be repeated to add more dnodes in the cluster.
 
 Any node that is in the cluster and online can be the firstEp of new nodes.
 Nodes use the firstEp parameter only when joining a cluster for the first time. After a node has joined the cluster, it stores the latest mnode in its end point list and no longer makes use of firstEp.
-However, firstEp is used by clients that connect to the cluster. For example, if you run `taos shell` without arguments, it connects to the firstEp by default.
+
+However, firstEp is used by clients that connect to the cluster. For example, if you run TDengine CLI `taos` without arguments, it connects to the firstEp by default.
+
 Two dnodes that are launched without a firstEp value operate independently of each other. It is not possible to add one dnode to the other dnode and form a cluster. It is also not possible to form two independent clusters into a new cluster.
 
 :::

docs/en/14-reference/11-docker/index.md

@@ -72,7 +72,7 @@ Next, ensure the hostname "tdengine" is resolvable in `/etc/hosts`.
 echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts
 ```
 
-Finally, the TDengine service can be accessed from the taos shell or any connector with "tdengine" as the server address.
+Finally, the TDengine service can be accessed from the TDengine CLI or any connector with "tdengine" as the server address.
 
 ```shell
 taos -h tdengine -P 6030

docs/en/14-reference/12-config/index.md

@@ -344,7 +344,7 @@ The charset that takes effect is UTF-8.
 | Attribute     | Description                            |
 | -------- | --------------------------------- |
 | Applicable    | Server and Client         |
-| Meaning       | The interval for taos shell to send heartbeat to mnode |
+| Meaning       | The interval for TDengine CLI to send heartbeat to mnode |
 | Unit          | second                                          |
 | Value Range   | 1-120                                         |
 | Default Value | 3                              |

docs/examples/java/src/main/java/com/taos/example/RestInsertExample.java

@@ -16,14 +16,14 @@ public class RestInsertExample {
 
     private static List<String> getRawData() {
         return Arrays.asList(
-                "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,California.SanFrancisco,2",
-                "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,California.SanFrancisco,2",
-                "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,California.SanFrancisco,2",
-                "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,California.SanFrancisco,3",
-                "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,California.LosAngeles,2",
-                "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,California.LosAngeles,2",
-                "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,California.LosAngeles,3",
-                "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,California.LosAngeles,3"
+                "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,'California.SanFrancisco',2",
+                "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,'California.SanFrancisco',2",
+                "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,'California.SanFrancisco',2",
+                "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,'California.SanFrancisco',3",
+                "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,'California.LosAngeles',2",
+                "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,'California.LosAngeles',2",
+                "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,'California.LosAngeles',3",
+                "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,'California.LosAngeles',3"
         );
     }
 

docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java

@@ -57,7 +57,7 @@ public class SubscribeDemo {
             properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
             properties.setProperty(TMQConstants.GROUP_ID, "test");
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
-                    "com.taosdata.jdbc.MetersDeserializer");
+                    "com.taos.example.MetersDeserializer");
 
             // poll data
             try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {

docs/examples/java/src/main/java/com/taos/example/highvolume/DataBaseMonitor.java

+package com.taos.example.highvolume;
+
+import java.sql.*;
+
+/**
+ * Prepare target database.
+ * Count total records in database periodically so that we can estimate the writing speed.
+ */
+public class DataBaseMonitor {
+    private Connection conn;
+    private Statement stmt;
+
+    public DataBaseMonitor init() throws SQLException {
+        if (conn == null) {
+            String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+            conn = DriverManager.getConnection(jdbcURL);
+            stmt = conn.createStatement();
+        }
+        return this;
+    }
+
+    public void close() {
+        try {
+            stmt.close();
+        } catch (SQLException e) {
+        }
+        try {
+            conn.close();
+        } catch (SQLException e) {
+        }
+    }
+
+    public void prepareDatabase() throws SQLException {
+        stmt.execute("DROP DATABASE IF EXISTS test");
+        stmt.execute("CREATE DATABASE test");
+        stmt.execute("CREATE STABLE test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
+    }
+
+    public Long count() throws SQLException {
+        if (!stmt.isClosed()) {
+            ResultSet result = stmt.executeQuery("SELECT count(*) from test.meters");
+            result.next();
+            return result.getLong(1);
+        }
+        return null;
+    }
+
+    /**
+     * show test.stables;
+     *
+     *               name              |      created_time       | columns |  tags  |   tables    |
+     * ============================================================================================
+     *  meters                         | 2022-07-20 08:39:30.902 |       4 |      2 |      620000 |
+     */
+    public  Long getTableCount() throws SQLException {
+        if (!stmt.isClosed()) {
+            ResultSet result = stmt.executeQuery("show test.stables");
+            result.next();
+            return result.getLong(5);
+        }
+        return null;
+    }
+}
\ No newline at end of file

docs/examples/java/src/main/java/com/taos/example/highvolume/FastWriteExample.java

+package com.taos.example.highvolume;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
+
+public class FastWriteExample {
+    final static Logger logger = LoggerFactory.getLogger(FastWriteExample.class);
+
+    final static int taskQueueCapacity = 1000000;
+    final static List<BlockingQueue<String>> taskQueues = new ArrayList<>();
+    final static List<ReadTask> readTasks = new ArrayList<>();
+    final static List<WriteTask> writeTasks = new ArrayList<>();
+    final static DataBaseMonitor databaseMonitor = new DataBaseMonitor();
+
+    public static void stopAll() {
+        logger.info("shutting down");
+        readTasks.forEach(task -> task.stop());
+        writeTasks.forEach(task -> task.stop());
+        databaseMonitor.close();
+    }
+
+    public static void main(String[] args) throws InterruptedException, SQLException {
+        int readTaskCount = args.length > 0 ? Integer.parseInt(args[0]) : 1;
+        int writeTaskCount = args.length > 1 ? Integer.parseInt(args[1]) : 3;
+        int tableCount = args.length > 2 ? Integer.parseInt(args[2]) : 1000;
+        int maxBatchSize = args.length > 3 ? Integer.parseInt(args[3]) : 3000;
+
+        logger.info("readTaskCount={}, writeTaskCount={} tableCount={} maxBatchSize={}",
+                readTaskCount, writeTaskCount, tableCount, maxBatchSize);
+
+        databaseMonitor.init().prepareDatabase();
+
+        // Create task queues, whiting tasks and start writing threads.
+        for (int i = 0; i < writeTaskCount; ++i) {
+            BlockingQueue<String> queue = new ArrayBlockingQueue<>(taskQueueCapacity);
+            taskQueues.add(queue);
+            WriteTask task = new WriteTask(queue, maxBatchSize);
+            Thread t = new Thread(task);
+            t.setName("WriteThread-" + i);
+            t.start();
+        }
+
+        // create reading tasks and start reading threads
+        int tableCountPerTask = tableCount / readTaskCount;
+        for (int i = 0; i < readTaskCount; ++i) {
+            ReadTask task = new ReadTask(i, taskQueues, tableCountPerTask);
+            Thread t = new Thread(task);
+            t.setName("ReadThread-" + i);
+            t.start();
+        }
+
+        Runtime.getRuntime().addShutdownHook(new Thread(FastWriteExample::stopAll));
+
+        long lastCount = 0;
+        while (true) {
+            Thread.sleep(10000);
+            long numberOfTable = databaseMonitor.getTableCount();
+            long count = databaseMonitor.count();
+            logger.info("numberOfTable={} count={} speed={}", numberOfTable, count, (count - lastCount) / 10);
+            lastCount = count;
+        }
+    }
+}
\ No newline at end of file

docs/examples/java/src/main/java/com/taos/example/highvolume/MockDataSource.java

+package com.taos.example.highvolume;
+
+import java.util.Iterator;
+
+/**
+ * Generate test data
+ */
+class MockDataSource implements Iterator {
+    private String tbNamePrefix;
+    private int tableCount;
+    private long maxRowsPerTable = 1000000000L;
+
+    // 100 milliseconds between two neighbouring rows.
+    long startMs = System.currentTimeMillis() - maxRowsPerTable * 100;
+    private int currentRow = 0;
+    private int currentTbId = -1;
+
+    // mock values
+    String[] location = {"LosAngeles", "SanDiego", "Hollywood", "Compton", "San Francisco"};
+    float[] current = {8.8f, 10.7f, 9.9f, 8.9f, 9.4f};
+    int[] voltage = {119, 116, 111, 113, 118};
+    float[] phase = {0.32f, 0.34f, 0.33f, 0.329f, 0.141f};
+
+    public MockDataSource(String tbNamePrefix, int tableCount) {
+        this.tbNamePrefix = tbNamePrefix;
+        this.tableCount = tableCount;
+    }
+
+    @Override
+    public boolean hasNext() {
+        currentTbId += 1;
+        if (currentTbId == tableCount) {
+            currentTbId = 0;
+            currentRow += 1;
+        }
+        return currentRow < maxRowsPerTable;
+    }
+
+    @Override
+    public String next() {
+        long ts = startMs + 100 * currentRow;
+        int groupId = currentTbId % 5 == 0 ? currentTbId / 5 : currentTbId / 5 + 1;
+        StringBuilder sb = new StringBuilder(tbNamePrefix + "_" + currentTbId + ","); // tbName
+        sb.append(ts).append(','); // ts
+        sb.append(current[currentRow % 5]).append(','); // current
+        sb.append(voltage[currentRow % 5]).append(','); // voltage
+        sb.append(phase[currentRow % 5]).append(','); // phase
+        sb.append(location[currentRow % 5]).append(','); // location
+        sb.append(groupId); // groupID
+
+        return sb.toString();
+    }
+}
\ No newline at end of file

docs/examples/java/src/main/java/com/taos/example/highvolume/ReadTask.java

+package com.taos.example.highvolume;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+class ReadTask implements Runnable {
+    private final static Logger logger = LoggerFactory.getLogger(ReadTask.class);
+    private final int taskId;
+    private final List<BlockingQueue<String>> taskQueues;
+    private final int queueCount;
+    private final int tableCount;
+    private boolean active = true;
+
+    public ReadTask(int readTaskId, List<BlockingQueue<String>> queues, int tableCount) {
+        this.taskId = readTaskId;
+        this.taskQueues = queues;
+        this.queueCount = queues.size();
+        this.tableCount = tableCount;
+    }
+
+    /**
+     * Assign data received to different queues.
+     * Here we use the suffix number in table name.
+     * You are expected to define your own rule in practice.
+     *
+     * @param line record received
+     * @return which queue to use
+     */
+    public int getQueueId(String line) {
+        String tbName = line.substring(0, line.indexOf(',')); // For example: tb1_101
+        String suffixNumber = tbName.split("_")[1];
+        return Integer.parseInt(suffixNumber) % this.queueCount;
+    }
+
+    @Override
+    public void run() {
+        logger.info("started");
+        Iterator<String> it = new MockDataSource("tb" + this.taskId, tableCount);
+        try {
+            while (it.hasNext() && active) {
+                String line = it.next();
+                int queueId = getQueueId(line);
+                taskQueues.get(queueId).put(line);
+            }
+        } catch (Exception e) {
+            logger.error("Read Task Error", e);
+        }
+    }
+
+    public void stop() {
+        logger.info("stop");
+        this.active = false;
+    }
+}
\ No newline at end of file

docs/examples/java/src/main/java/com/taos/example/highvolume/SQLWriter.java

+package com.taos.example.highvolume;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A helper class encapsulate the logic of writing using SQL.
+ * <p>
+ * The main interfaces are two methods:
+ * <ol>
+ *     <li>{@link SQLWriter#processLine}, which receive raw lines from WriteTask and group them by table names.</li>
+ *     <li>{@link  SQLWriter#flush}, which assemble INSERT statement and execute it.</li>
+ * </ol>
+ * <p>
+ * There is a technical skill worth mentioning: we create table as needed when "table does not exist" error occur instead of creating table automatically using syntax "INSET INTO tb USING stb".
+ * This ensure that checking table existence is a one-time-only operation.
+ * </p>
+ *
+ * </p>
+ */
+public class SQLWriter {
+    final static Logger logger = LoggerFactory.getLogger(SQLWriter.class);
+
+    private Connection conn;
+    private Statement stmt;
+
+    /**
+     * current number of buffered records
+     */
+    private int bufferedCount = 0;
+    /**
+     * Maximum number of buffered records.
+     * Flush action will be triggered if bufferedCount reached this value,
+     */
+    private int maxBatchSize;
+
+
+    /**
+     * Maximum SQL length.
+     */
+    private int maxSQLLength;
+
+    /**
+     * Map from table name to column values. For example:
+     * "tb001" -> "(1648432611249,2.1,114,0.09) (1648432611250,2.2,135,0.2)"
+     */
+    private Map<String, String> tbValues = new HashMap<>();
+
+    /**
+     * Map from table name to tag values in the same order as creating stable.
+     * Used for creating table.
+     */
+    private Map<String, String> tbTags = new HashMap<>();
+
+    public SQLWriter(int maxBatchSize) {
+        this.maxBatchSize = maxBatchSize;
+    }
+
+
+    /**
+     * Get Database Connection
+     *
+     * @return Connection
+     * @throws SQLException
+     */
+    private static Connection getConnection() throws SQLException {
+        String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+        return DriverManager.getConnection(jdbcURL);
+    }
+
+    /**
+     * Create Connection and Statement
+     *
+     * @throws SQLException
+     */
+    public void init() throws SQLException {
+        conn = getConnection();
+        stmt = conn.createStatement();
+        stmt.execute("use test");
+        ResultSet rs = stmt.executeQuery("show variables");
+        while (rs.next()) {
+            String configName = rs.getString(1);
+            if ("maxSQLLength".equals(configName)) {
+                maxSQLLength = Integer.parseInt(rs.getString(2));
+                logger.info("maxSQLLength={}", maxSQLLength);
+            }
+        }
+    }
+
+    /**
+     * Convert raw data to SQL fragments, group them by table name and cache them in a HashMap.
+     * Trigger writing when number of buffered records reached maxBachSize.
+     *
+     * @param line raw data get from task queue in format: tbName,ts,current,voltage,phase,location,groupId
+     */
+    public void processLine(String line) throws SQLException {
+        bufferedCount += 1;
+        int firstComma = line.indexOf(',');
+        String tbName = line.substring(0, firstComma);
+        int lastComma = line.lastIndexOf(',');
+        int secondLastComma = line.lastIndexOf(',', lastComma - 1);
+        String value = "(" + line.substring(firstComma + 1, secondLastComma) + ") ";
+        if (tbValues.containsKey(tbName)) {
+            tbValues.put(tbName, tbValues.get(tbName) + value);
+        } else {
+            tbValues.put(tbName, value);
+        }
+        if (!tbTags.containsKey(tbName)) {
+            String location = line.substring(secondLastComma + 1, lastComma);
+            String groupId = line.substring(lastComma + 1);
+            String tagValues = "('" + location + "'," + groupId + ')';
+            tbTags.put(tbName, tagValues);
+        }
+        if (bufferedCount == maxBatchSize) {
+            flush();
+        }
+    }
+
+
+    /**
+     * Assemble INSERT statement using buffered SQL fragments in Map {@link SQLWriter#tbValues} and execute it.
+     * In case of "Table does not exit" exception, create all tables in the sql and retry the sql.
+     */
+    public void flush() throws SQLException {
+        StringBuilder sb = new StringBuilder("INSERT INTO ");
+        for (Map.Entry<String, String> entry : tbValues.entrySet()) {
+            String tableName = entry.getKey();
+            String values = entry.getValue();
+            String q = tableName + " values " + values + " ";
+            if (sb.length() + q.length() > maxSQLLength) {
+                executeSQL(sb.toString());
+                logger.warn("increase maxSQLLength or decrease maxBatchSize to gain better performance");
+                sb = new StringBuilder("INSERT INTO ");
+            }
+            sb.append(q);
+        }
+        executeSQL(sb.toString());
+        tbValues.clear();
+        bufferedCount = 0;
+    }
+
+    private void executeSQL(String sql) throws SQLException {
+        try {
+            stmt.executeUpdate(sql);
+        } catch (SQLException e) {
+            // convert to error code defined in taoserror.h
+            int errorCode = e.getErrorCode() & 0xffff;
+            if (errorCode == 0x362 || errorCode == 0x218) {
+                // Table does not exist
+                createTables();
+                executeSQL(sql);
+            } else {
+                logger.error("Execute SQL: {}", sql);
+                throw e;
+            }
+        } catch (Throwable throwable) {
+            logger.error("Execute SQL: {}", sql);
+            throw throwable;
+        }
+    }
+
+    /**
+     * Create tables in batch using syntax:
+     * <p>
+     * CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) [IF NOT EXISTS] tb_name2 USING stb_name TAGS (tag_value2, ...) ...;
+     * </p>
+     */
+    private void createTables() throws SQLException {
+        StringBuilder sb = new StringBuilder("CREATE TABLE ");
+        for (String tbName : tbValues.keySet()) {
+            String tagValues = tbTags.get(tbName);
+            sb.append("IF NOT EXISTS ").append(tbName).append(" USING meters TAGS ").append(tagValues).append(" ");
+        }
+        String sql = sb.toString();
+        try {
+            stmt.executeUpdate(sql);
+        } catch (Throwable throwable) {
+            logger.error("Execute SQL: {}", sql);
+            throw throwable;
+        }
+    }
+
+    public boolean hasBufferedValues() {
+        return bufferedCount > 0;
+    }
+
+    public int getBufferedCount() {
+        return bufferedCount;
+    }
+
+    public void close() {
+        try {
+            stmt.close();
+        } catch (SQLException e) {
+        }
+        try {
+            conn.close();
+        } catch (SQLException e) {
+        }
+    }
+}
\ No newline at end of file

docs/examples/java/src/main/java/com/taos/example/highvolume/StmtWriter.java

+package com.taos.example.highvolume;
+
+public class StmtWriter {
+}

docs/examples/java/src/main/java/com/taos/example/highvolume/WriteTask.java

+package com.taos.example.highvolume;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+
+class WriteTask implements Runnable {
+    private final static Logger logger = LoggerFactory.getLogger(WriteTask.class);
+    private final int maxBatchSize;
+
+    // the queue from which this writing task get raw data.
+    private final BlockingQueue<String> queue;
+
+    // A flag indicate whether to continue.
+    private boolean active = true;
+
+    public WriteTask(BlockingQueue<String> taskQueue, int maxBatchSize) {
+        this.queue = taskQueue;
+        this.maxBatchSize = maxBatchSize;
+    }
+
+    @Override
+    public void run() {
+        logger.info("started");
+        String line = null; // data getting from the queue just now.
+        SQLWriter writer = new SQLWriter(maxBatchSize);
+        try {
+            writer.init();
+            while (active) {
+                line = queue.poll();
+                if (line != null) {
+                    // parse raw data and buffer the data.
+                    writer.processLine(line);
+                } else if (writer.hasBufferedValues()) {
+                    // write data immediately if no more data in the queue
+                    writer.flush();
+                } else {
+                    // sleep a while to avoid high CPU usage if no more data in the queue and no buffered records, .
+                    Thread.sleep(100);
+                }
+            }
+            if (writer.hasBufferedValues()) {
+                writer.flush();
+            }
+        } catch (Exception e) {
+            String msg = String.format("line=%s, bufferedCount=%s", line, writer.getBufferedCount());
+            logger.error(msg, e);
+        } finally {
+            writer.close();
+        }
+    }
+
+    public void stop() {
+        logger.info("stop");
+        this.active = false;
+    }
+}
\ No newline at end of file

docs/examples/java/src/test/java/com/taos/test/TestAll.java

@@ -23,16 +23,16 @@ public class TestAll {
         String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
         try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
             try (Statement stmt = conn.createStatement()) {
-                String sql = "INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES('2018-10-03 14:38:05.000',10.30000,219,0.31000)\n" +
-                        "                        power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES('2018-10-03 15:38:15.000',12.60000,218,0.33000)\n" +
-                        "                        power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES('2018-10-03 15:38:16.800',12.30000,221,0.31000)\n" +
-                        "                        power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES('2018-10-03 15:38:16.650',10.30000,218,0.25000)\n" +
-                        "                        power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES('2018-10-03 15:38:05.500',11.80000,221,0.28000)\n" +
-                        "                        power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES('2018-10-03 15:38:16.600',13.40000,223,0.29000)\n" +
-                        "                        power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES('2018-10-03 15:38:05.000',10.80000,223,0.29000)\n" +
-                        "                        power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES('2018-10-03 15:38:06.000',10.80000,223,0.29000)\n" +
-                        "                        power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES('2018-10-03 15:38:07.000',10.80000,223,0.29000)\n" +
-                        "                        power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES('2018-10-03 15:38:08.500',11.50000,221,0.35000)";
+                String sql = "INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-03 14:38:05.000',10.30000,219,0.31000)\n" +
+                        "                        power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-03 15:38:15.000',12.60000,218,0.33000)\n" +
+                        "                        power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-03 15:38:16.800',12.30000,221,0.31000)\n" +
+                        "                        power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 15:38:16.650',10.30000,218,0.25000)\n" +
+                        "                        power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 15:38:05.500',11.80000,221,0.28000)\n" +
+                        "                        power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 15:38:16.600',13.40000,223,0.29000)\n" +
+                        "                        power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 15:38:05.000',10.80000,223,0.29000)\n" +
+                        "                        power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 15:38:06.000',10.80000,223,0.29000)\n" +
+                        "                        power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 15:38:07.000',10.80000,223,0.29000)\n" +
+                        "                        power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 15:38:08.500',11.50000,221,0.35000)";
 
                 stmt.execute(sql);
             }

docs/examples/python/fast_write_example.py

+# install dependencies:
+# recommend python >= 3.8
+# pip3 install faster-fifo
+#
+
+import logging
+import math
+import sys
+import time
+import os
+from multiprocessing import Process
+from faster_fifo import Queue
+from mockdatasource import MockDataSource
+from queue import Empty
+from typing import List
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(asctime)s [%(name)s] - %(message)s")
+
+READ_TASK_COUNT = 1
+WRITE_TASK_COUNT = 1
+TABLE_COUNT = 1000
+QUEUE_SIZE = 1000000
+MAX_BATCH_SIZE = 3000
+
+read_processes = []
+write_processes = []
+
+
+def get_connection():
+    """
+    If variable TDENGINE_FIRST_EP is provided then it will be used. If not, firstEP in /etc/taos/taos.cfg will be used.
+    You can also override the default username and password by supply variable TDENGINE_USER and TDENGINE_PASSWORD
+    """
+    import taos
+    firstEP = os.environ.get("TDENGINE_FIRST_EP")
+    if firstEP:
+        host, port = firstEP.split(":")
+    else:
+        host, port = None, 0
+    user = os.environ.get("TDENGINE_USER", "root")
+    password = os.environ.get("TDENGINE_PASSWORD", "taosdata")
+    return taos.connect(host=host, port=int(port), user=user, password=password)
+
+
+# ANCHOR: read
+
+def run_read_task(task_id: int, task_queues: List[Queue]):
+    table_count_per_task = TABLE_COUNT // READ_TASK_COUNT
+    data_source = MockDataSource(f"tb{task_id}", table_count_per_task)
+    try:
+        for batch in data_source:
+            for table_id, rows in batch:
+                # hash data to different queue
+                i = table_id % len(task_queues)
+                # block putting forever when the queue is full
+                task_queues[i].put_many(rows, block=True, timeout=-1)
+    except KeyboardInterrupt:
+        pass
+
+
+# ANCHOR_END: read
+
+# ANCHOR: write
+def run_write_task(task_id: int, queue: Queue):
+    from sql_writer import SQLWriter
+    log = logging.getLogger(f"WriteTask-{task_id}")
+    writer = SQLWriter(get_connection)
+    lines = None
+    try:
+        while True:
+            try:
+                # get as many as possible
+                lines = queue.get_many(block=False, max_messages_to_get=MAX_BATCH_SIZE)
+                writer.process_lines(lines)
+            except Empty:
+                time.sleep(0.01)
+    except KeyboardInterrupt:
+        pass
+    except BaseException as e:
+        log.debug(f"lines={lines}")
+        raise e
+
+
+# ANCHOR_END: write
+
+def set_global_config():
+    argc = len(sys.argv)
+    if argc > 1:
+        global READ_TASK_COUNT
+        READ_TASK_COUNT = int(sys.argv[1])
+    if argc > 2:
+        global WRITE_TASK_COUNT
+        WRITE_TASK_COUNT = int(sys.argv[2])
+    if argc > 3:
+        global TABLE_COUNT
+        TABLE_COUNT = int(sys.argv[3])
+    if argc > 4:
+        global QUEUE_SIZE
+        QUEUE_SIZE = int(sys.argv[4])
+    if argc > 5:
+        global MAX_BATCH_SIZE
+        MAX_BATCH_SIZE = int(sys.argv[5])
+
+
+# ANCHOR: monitor
+def run_monitor_process():
+    log = logging.getLogger("DataBaseMonitor")
+    conn = get_connection()
+    conn.execute("DROP DATABASE IF EXISTS test")
+    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE STABLE test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
+                 "TAGS (location BINARY(64), groupId INT)")
+
+    def get_count():
+        res = conn.query("SELECT count(*) FROM test.meters")
+        rows = res.fetch_all()
+        return rows[0][0] if rows else 0
+
+    last_count = 0
+    while True:
+        time.sleep(10)
+        count = get_count()
+        log.info(f"count={count} speed={(count - last_count) / 10}")
+        last_count = count
+
+
+# ANCHOR_END: monitor
+# ANCHOR: main
+def main():
+    set_global_config()
+    logging.info(f"READ_TASK_COUNT={READ_TASK_COUNT}, WRITE_TASK_COUNT={WRITE_TASK_COUNT}, "
+                 f"TABLE_COUNT={TABLE_COUNT}, QUEUE_SIZE={QUEUE_SIZE}, MAX_BATCH_SIZE={MAX_BATCH_SIZE}")
+
+    monitor_process = Process(target=run_monitor_process)
+    monitor_process.start()
+    time.sleep(3)  # waiting for database ready.
+
+    task_queues: List[Queue] = []
+    # create task queues
+    for i in range(WRITE_TASK_COUNT):
+        queue = Queue(max_size_bytes=QUEUE_SIZE)
+        task_queues.append(queue)
+
+    # create write processes
+    for i in range(WRITE_TASK_COUNT):
+        p = Process(target=run_write_task, args=(i, task_queues[i]))
+        p.start()
+        logging.debug(f"WriteTask-{i} started with pid {p.pid}")
+        write_processes.append(p)
+
+    # create read processes
+    for i in range(READ_TASK_COUNT):
+        queues = assign_queues(i, task_queues)
+        p = Process(target=run_read_task, args=(i, queues))
+        p.start()
+        logging.debug(f"ReadTask-{i} started with pid {p.pid}")
+        read_processes.append(p)
+
+    try:
+        monitor_process.join()
+    except KeyboardInterrupt:
+        monitor_process.terminate()
+        [p.terminate() for p in read_processes]
+        [p.terminate() for p in write_processes]
+        [q.close() for q in task_queues]
+
+
+def assign_queues(read_task_id, task_queues):
+    """
+    Compute target queues for a specific read task.
+    """
+    ratio = WRITE_TASK_COUNT / READ_TASK_COUNT
+    from_index = math.floor(read_task_id * ratio)
+    end_index = math.ceil((read_task_id + 1) * ratio)
+    return task_queues[from_index:end_index]
+
+
+if __name__ == '__main__':
+    main()
+# ANCHOR_END: main

docs/examples/python/mockdatasource.py

+import time
+
+
+class MockDataSource:
+    samples = [
+        "8.8,119,0.32,LosAngeles,0",
+        "10.7,116,0.34,SanDiego,1",
+        "9.9,111,0.33,Hollywood,2",
+        "8.9,113,0.329,Compton,3",
+        "9.4,118,0.141,San Francisco,4"
+    ]
+
+    def __init__(self, tb_name_prefix, table_count):
+        self.table_name_prefix = tb_name_prefix + "_"
+        self.table_count = table_count
+        self.max_rows = 10000000
+        self.current_ts = round(time.time() * 1000) - self.max_rows * 100
+        # [(tableId, tableName, values),]
+        self.data = self._init_data()
+
+    def _init_data(self):
+        lines = self.samples * (self.table_count // 5 + 1)
+        data = []
+        for i in range(self.table_count):
+            table_name = self.table_name_prefix + str(i)
+            data.append((i, table_name, lines[i]))  # tableId, row
+        return data
+
+    def __iter__(self):
+        self.row = 0
+        return self
+
+    def __next__(self):
+        """
+        next 1000 rows for each table.
+        return: {tableId:[row,...]}
+        """
+        # generate 1000 timestamps
+        ts = []
+        for _ in range(1000):
+            self.current_ts += 100
+            ts.append(str(self.current_ts))
+        # add timestamp to each row
+        # [(tableId, ["tableName,ts,current,voltage,phase,location,groupId"])]
+        result = []
+        for table_id, table_name, values in self.data:
+            rows = [table_name + ',' + t + ',' + values for t in ts]
+            result.append((table_id, rows))
+        return result

docs/examples/python/sql_writer.py

+import logging
+import taos
+
+
+class SQLWriter:
+    log = logging.getLogger("SQLWriter")
+
+    def __init__(self, get_connection_func):
+        self._tb_values = {}
+        self._tb_tags = {}
+        self._conn = get_connection_func()
+        self._max_sql_length = self.get_max_sql_length()
+        self._conn.execute("USE test")
+
+    def get_max_sql_length(self):
+        rows = self._conn.query("SHOW variables").fetch_all()
+        for r in rows:
+            name = r[0]
+            if name == "maxSQLLength":
+                return int(r[1])
+        return 1024 * 1024
+
+    def process_lines(self, lines: str):
+        """
+        :param lines: [[tbName,ts,current,voltage,phase,location,groupId]]
+        """
+        for line in lines:
+            ps = line.split(",")
+            table_name = ps[0]
+            value = '(' + ",".join(ps[1:-2]) + ') '
+            if table_name in self._tb_values:
+                self._tb_values[table_name] += value
+            else:
+                self._tb_values[table_name] = value
+
+            if table_name not in self._tb_tags:
+                location = ps[-2]
+                group_id = ps[-1]
+                tag_value = f"('{location}',{group_id})"
+                self._tb_tags[table_name] = tag_value
+        self.flush()
+
+    def flush(self):
+        """
+        Assemble INSERT statement and execute it.
+        When the sql length grows close to MAX_SQL_LENGTH, the sql will be executed immediately, and a new INSERT statement will be created.
+        In case of "Table does not exit" exception, tables in the sql will be created and the sql will be re-executed.
+        """
+        sql = "INSERT INTO "
+        sql_len = len(sql)
+        buf = []
+        for tb_name, values in self._tb_values.items():
+            q = tb_name + " VALUES " + values
+            if sql_len + len(q) >= self._max_sql_length:
+                sql += " ".join(buf)
+                self.execute_sql(sql)
+                sql = "INSERT INTO "
+                sql_len = len(sql)
+                buf = []
+            buf.append(q)
+            sql_len += len(q)
+        sql += " ".join(buf)
+        self.execute_sql(sql)
+        self._tb_values.clear()
+
+    def execute_sql(self, sql):
+        try:
+            self._conn.execute(sql)
+        except taos.Error as e:
+            error_code = e.errno & 0xffff
+            # Table does not exit
+            if error_code == 9731:
+                self.create_tables()
+            else:
+                self.log.error("Execute SQL: %s", sql)
+                raise e
+        except BaseException as baseException:
+            self.log.error("Execute SQL: %s", sql)
+            raise baseException
+
+    def create_tables(self):
+        sql = "CREATE TABLE "
+        for tb in self._tb_values.keys():
+            tag_values = self._tb_tags[tb]
+            sql += "IF NOT EXISTS " + tb + " USING meters TAGS " + tag_values + " "
+        try:
+            self._conn.execute(sql)
+        except BaseException as e:
+            self.log.error("Execute SQL: %s", sql)
+            raise e

docs/zh/07-develop/03-insert-data/01-sql-writing.mdx

@@ -23,7 +23,7 @@ import PhpStmt from "./_php_stmt.mdx";
 
 ## SQL 写入简介
 
-应用通过连接器执行 INSERT 语句来插入数据，用户还可以通过 TAOS Shell，手动输入 INSERT 语句插入数据。
+应用通过连接器执行 INSERT 语句来插入数据，用户还可以通过 TDengine CLI，手动输入 INSERT 语句插入数据。
 
 ### 一次写入一条
 下面这条 INSERT 就将一条记录写入到表 d1001 中：

docs/zh/07-develop/04-query-data/index.mdx

@@ -52,7 +52,7 @@ Query OK, 2 row(s) in set (0.001100s)
 
 ### 示例一
 
-在 TAOS Shell，查找加利福尼亚州所有智能电表采集的电压平均值，并按照 location 分组。
+在 TDengine CLI，查找加利福尼亚州所有智能电表采集的电压平均值，并按照 location 分组。
 
 ```
 taos> SELECT AVG(voltage), location FROM meters GROUP BY location;
@@ -65,7 +65,7 @@ Query OK, 2 rows in database (0.005995s)
 
 ### 示例二
 
-在 TAOS shell, 查找 groupId 为 2 的所有智能电表的记录条数，电流的最大值。
+在 TDengine CLI, 查找 groupId 为 2 的所有智能电表的记录条数，电流的最大值。
 
 ```
 taos> SELECT count(*), max(current) FROM meters where groupId = 2;

docs/zh/10-deployment/01-deploy.md

@@ -71,7 +71,7 @@ serverPort            6030
 
 ## 启动集群
 
-按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 taos shell，从 shell 里执行命令“SHOW DNODES”，如下所示：
+按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 TDengine CLI，在其中执行命令 “SHOW DNODES”，如下所示：
 
 ```
 taos> show dnodes;
@@ -115,7 +115,7 @@ SHOW DNODES;
 
 任何已经加入集群在线的数据节点，都可以作为后续待加入节点的 firstEp。
 firstEp 这个参数仅仅在该数据节点首次加入集群时有作用，加入集群后，该数据节点会保存最新的 mnode 的 End Point 列表，不再依赖这个参数。
-接下来，配置文件中的 firstEp 参数就主要在客户端连接的时候使用了，例如 taos shell 如果不加参数，会默认连接由 firstEp 指定的节点。
+接下来，配置文件中的 firstEp 参数就主要在客户端连接的时候使用了，例如 TDengine CLI 如果不加参数，会默认连接由 firstEp 指定的节点。
 两个没有配置 firstEp 参数的数据节点 dnode 启动后，会独立运行起来。这个时候，无法将其中一个数据节点加入到另外一个数据节点，形成集群。无法将两个独立的集群合并成为新的集群。
 
 :::

docs/zh/10-deployment/03-k8s.md

@@ -366,7 +366,7 @@ kubectl scale statefulsets tdengine --replicas=1
 
 ```
 
-在 taos shell 中的所有数据库操作将无法成功。
+在 TDengine CLI 中的所有数据库操作将无法成功。
 
 ```
 taos> show dnodes;

docs/zh/14-reference/11-docker/index.md

@@ -32,7 +32,7 @@ taos> show databases;
 Query OK, 2 rows in database (0.033802s)
 ```
 
-因为运行在容器中的 TDengine 服务端使用容器的 hostname 建立连接，使用 TDengine CLI 或者各种连接器（例如 JDBC-JNI）从容器外访问容器内的 TDengine 比较复杂，所以上述方式是访问容器中 TDengine 服务的最简单的方法，适用于一些简单场景。如果在一些复杂场景下想要从容器化使用 taos shell 或者各种连接器访问容器中的 TDengine 服务，请参考下一节。
+因为运行在容器中的 TDengine 服务端使用容器的 hostname 建立连接，使用 TDengine CLI 或者各种连接器（例如 JDBC-JNI）从容器外访问容器内的 TDengine 比较复杂，所以上述方式是访问容器中 TDengine 服务的最简单的方法，适用于一些简单场景。如果在一些复杂场景下想要从容器化使用 TDengine CLI 或者各种连接器访问容器中的 TDengine 服务，请参考下一节。
 
 ## 在 host 网络上启动 TDengine
 
@@ -75,7 +75,7 @@ docker run -d \
 echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts
 ```
 
-最后，可以从 taos shell 或者任意连接器以 "tdengine" 为服务端地址访问 TDengine 服务。
+最后，可以从 TDengine CLI 或者任意连接器以 "tdengine" 为服务端地址访问 TDengine 服务。
 
 ```shell
 taos -h tdengine -P 6030
@@ -354,7 +354,7 @@ test-docker_td-2_1   /tini -- /usr/bin/entrypoi ...   Up
 test-docker_td-3_1   /tini -- /usr/bin/entrypoi ...   Up
 ```
 
-4. 用 taos shell 查看 dnodes
+4. 用 TDengine CLI 查看 dnodes
 
 ```shell
 

docs/zh/17-operation/08-export.md

@@ -7,7 +7,7 @@ description: 如何导出 TDengine 中的数据
 
 ## 按表导出 CSV 文件
 
-如果用户需要导出一个表或一个 STable 中的数据，可在 taos shell 中运行：
+如果用户需要导出一个表或一个 STable 中的数据，可在 TDengine CLI 中运行：
 
 ```sql
 select * from <tb_name> >> data.csv;

docs/zh/27-train-faq/01-faq.md

@@ -116,7 +116,7 @@ charset UTF-8
 
 ### 9. 表名显示不全
 
-由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
+由于 TDengine CLI 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
 
 ### 10. 如何进行数据迁移？
 

examples/JDBC/JDBCDemo/README-jdbc-windows.md

@@ -129,7 +129,7 @@ https://www.taosdata.com/cn/all-downloads/
 192.168.236.136	td01
 ```
 
-配置完成后，在命令行内使用taos shell连接server端
+配置完成后，在命令行内使用TDengine CLI连接server端
 
 ```shell
 C:\TDengine>taos -h td01

examples/nodejs/README-win.md

@@ -35,7 +35,7 @@ Python 2.7.18
 
 下载地址：https://www.taosdata.com/cn/all-downloads/，选择一个合适的windows-client下载（client应该尽量与server端的版本保持一致）
 
-使用client的taos shell连接server
+使用client的TDengine CLI连接server
 
 ```shell
 >taos -h node5

packaging/cfg/taos.cfg

@@ -38,7 +38,7 @@
 # The interval of dnode reporting status to mnode
 # statusInterval            1
 
-# The interval for taos shell to send heartbeat to mnode
+# The interval for TDengine CLI to send heartbeat to mnode
 # shellActivityTimer        3
 
 # The minimum sliding window time, milli-second

packaging/docker/README.md

@@ -47,7 +47,7 @@ taos> show databases;
 Query OK, 1 row(s) in set (0.002843s)
 ```
 
-Since TDengine use container hostname to establish connections, it's a bit more complex to use taos shell and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use taos shell or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need.
+Since TDengine use container hostname to establish connections, it's a bit more complex to use TDengine CLI and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use TDengine CLI or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need.
 
 ### Start with host network
 
@@ -87,7 +87,7 @@ docker run -d \
 
 This command starts a docker container with TDengine server running and maps the container's TCP ports from 6030 to 6049 to the host's ports from 6030 to 6049 with TCP protocol and UDP ports range 6030-6039 to the host's UDP ports 6030-6039. If the host is already running TDengine server and occupying the same port(s), you need to map the container's port to a different unused port segment. (Please see TDengine 2.0 Port Description for details). In order to support TDengine clients accessing TDengine server services, both TCP and UDP ports need to be exposed by default(unless `rpcForceTcp` is set to `1`).
 
-If you want to use taos shell or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service.
+If you want to use TDengine CLI or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service.
 
 If you set the `TAOS_FQDN` to host's hostname, it will works as using `hosts` network like previous use case. Otherwise, like in `-e TAOS_FQDN=tdengine`, you can add the hostname record `tdengine` into `/etc/hosts` (use `127.0.0.1` here in host path, if use TDengine client/application in other hosts, you should set the right ip to the host eg. `192.168.10.1`(check the real ip in host with `hostname -i` or `ip route list default`) to make the TDengine endpoint resolvable):
 
@@ -391,7 +391,7 @@ test_td-1_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp,
 test_td-2_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
 ```
 
-Check dnodes with taos shell:
+Check dnodes with TDengine CLI:
 
 ```bash
 $ docker-compose exec td-1 taos -s "show dnodes"

tests/docs-examples-test/jdbc.sh

+#!/bin/bash
+
+pgrep taosd || taosd >> /dev/null 2>&1 &
+pgrep taosadapter || taosadapter >> /dev/null 2>&1 &
+cd ../../docs/examples/java
+
+mvn clean test > jdbc-out.log 2>&1
+tail -n 20 jdbc-out.log
+
+cases=`grep 'Tests run' jdbc-out.log | awk 'END{print $3}'`
+totalJDBCCases=`echo ${cases/%,}`
+failed=`grep 'Tests run' jdbc-out.log | awk 'END{print $5}'`
+JDBCFailed=`echo ${failed/%,}`
+error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'`
+JDBCError=`echo ${error/%,}`
+
+totalJDBCFailed=`expr $JDBCFailed + $JDBCError`
+totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed`
+
+if [ "$totalJDBCSuccess" -gt "0" ]; then
+  echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}"
+fi
+
+if [ "$totalJDBCFailed" -ne "0" ]; then
+  echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}"
+  exit 8
+fi
\ No newline at end of file

tests/script/tsim/query/crash_sql.sim

@@ -76,7 +76,7 @@ sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL,
 
 
 print ================ start query ======================
-print ================ SQL used to cause taosd or taos shell  crash
+print ================ SQL used to cause taosd or TDengine CLI  crash
 sql_error select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
 
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT