[TD-2771] feature: python taosdemo. add lock.

tests/examples/python/taosdemo/README.md

@@ -22,8 +22,8 @@ Author: Shuduo Sang <sangshuduo@gmail.com>
 	-M, --stable                      flag, Use super table. Default is no
 	-s, --stbname <stable prefix>     stable_prefix, STable prefix name. Default is 'st'
 	-Q, --query <DEFAULT | command>   query, Execute query command. set 'DEFAULT' means select * from each table
-	-T, --numOfThreads <number>        num_of_threads, The number of threads. Default is 1.
-	-P, --numOfProcesses <number>      num_of_processes, The number of threads. Default is 1.
+	-T, --threads <number>            num_of_threads, The number of threads. Default is 1.
+	-C, --processes <number>          num_of_processes, The number of threads. Default is 1.
 	-r, --batch <number>              num_of_records_per_req, The number of records per request. Default is 1000.
 	-t, --numOfTb <number>            num_of_tables, The number of tables. Default is 1.
 	-n, --numOfRec <number>           num_of_records_per_table, The number of records per table. Default is 1.

tests/examples/python/taosdemo/taosdemo.py

@@ -21,7 +21,7 @@ import json
 import random
 import time
 import datetime
-from multiprocessing import Process, Pool
+from multiprocessing import Process, Pool, Lock
 from multipledispatch import dispatch
 from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
 
@@ -124,28 +124,22 @@ def query_func(process: int, thread: int, cmd: str):
             restful_execute(
                 oneMoreHost, port, user, password, cmd)
     else:
-        v_print("%s", "Send to first host")
+        v_print("%s%s%s", "Send ", cmd, " to the host")
         if native:
-            cursor.execute(cmd)
+            pass
+#            cursor.execute(cmd)
         else:
             restful_execute(
                 host, port, user, password, cmd)
 
 
-def query_data_process(i: int, cmd: str):
+def query_data_process(q_lock, i: int, cmd: str):
+    time.sleep(0.01)
     v_print("Process:%d threads: %d cmd: %s", i, threads, cmd)
 
-    with ThreadPoolExecutor(max_workers=threads) as executor:
-        workers = [
-            executor.submit(
-                query_func,
-                i,
-                j,
-                cmd) for j in range(
-                0,
-                threads)]
-
-        wait(workers, return_when=ALL_COMPLETED)
+    q_lock.aquire()
+    cursor_p.execute(cmd)
+    q_lock.release()
 
     return i
 
@@ -153,15 +147,18 @@ def query_data_process(i: int, cmd: str):
 def query_data(cmd: str):
     v_print("query_data processes: %d, cmd: %s", processes, cmd)
 
+    q_lock = Lock()
+
     pool = Pool(processes)
     for i in range(processes):
-        pool.apply_async(query_data_process, args=(i, cmd))
-        time.sleep(1)
+        pool.apply_async(query_data_process, args=(q_lock, i, cmd))
+#        time.sleep(1)
     pool.close()
     pool.join()
 
 
 def insert_data(processes: int):
+    i_lock = Lock()
     pool = Pool(processes)
 
     begin = 0
@@ -179,6 +176,7 @@ def insert_data(processes: int):
         quotient,
         remainder)
 
+    print("CBD LN210 processes:%d" % processes)
     for i in range(processes):
         begin = end
 
@@ -187,8 +185,8 @@ def insert_data(processes: int):
         else:
             end = begin + quotient
 
-        v_print("Process %d from %d to %d", i, begin, end)
-        pool.apply_async(insert_data_process, args=(i, begin, end))
+        v_print("insert_data Process %d from %d to %d", i, begin, end)
+        pool.apply_async(insert_data_process, args=(i_lock, i, begin, end))
 
     pool.close()
     pool.join()
@@ -295,10 +293,11 @@ def insert_func(process: int, thread: int):
 
                 sqlCmd.append("VALUES ")
                 for batchIter in range(0, batch):
-                    sqlCmd.append("('%s', %f) " %
-                                  (start_time +
-                                   datetime.timedelta(
-                                       milliseconds=batchIter),
+                    sqlCmd.append("(now, %f) " %
+                                  (
+#                                  start_time +
+#                                   datetime.timedelta(
+#                                       milliseconds=batchIter),
                                       random.random()))
                     row = row + 1
                     if row >= numOfRec:
@@ -310,18 +309,23 @@ def insert_func(process: int, thread: int):
 
             cmd = ' '.join(sqlCmd)
 
+            print("CBD: LN313")
             if measure:
                 exec_start_time = datetime.datetime.now()
 
+            print("CBD: LN316 native: %d" % native)
             if native:
-                v_print("insert_func - cursor:%x cmd:%s", hex(id(cursor)), cmd)
-                cursor.execute("SHOW DATABASES" )
-#                cursor.execute("%s" % cmd)
-                v_print("insert_func - cursor:%x cmd:%s done", hex(id(cursor)), cmd)
+                print("CBD: LN319: %s" % cmd)
+                print("conn: %s" % str(conn.__class__))
+                print("CBD: LN320 cursor:%d %s" % (id(cursor), str(cursor.__class__)))
+#                cursor.execute("SHOW DATABASES" )
+                affectedRows = cursor.execute(cmd)
+                print("CBD: LN323 affectedRows:%d" % affectedRows)
             else:
                 restful_execute(
                     host, port, user, password, cmd)
 
+            print("CBD: LN327")
             if measure:
                 exec_end_time = datetime.datetime.now()
                 exec_delta = exec_end_time - exec_start_time
@@ -363,9 +367,11 @@ def create_tb():
                     (tbName, j))
 
 
-def insert_data_process(i: int, begin: int, end: int):
+def insert_data_process(lock, i: int, begin: int, end: int):
+    print("CBD insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks)
+    time.sleep(1)
     tasks = end - begin
-    v_print("Process:%d table from %d to %d, tasks %d", i, begin, end, tasks)
+    i_lock.aquire()
 
     if (threads < (end - begin)):
         for j in range(begin, end, threads):
@@ -389,6 +395,7 @@ def insert_data_process(i: int, begin: int, end: int):
                     begin,
                     end)]
             wait(workers, return_when=ALL_COMPLETED)
+    i_lock.release()
 
 
 def printConfig():
@@ -457,7 +464,7 @@ if __name__ == "__main__":
     threads = 1
     insertOnly = False
     autosubtable = False
-    queryCmd = "select * from "
+    queryCmd = "DEFAULT"
     outOfOrder = 0
     rateOOOO = 0
     deleteMethod = 0
@@ -465,10 +472,10 @@ if __name__ == "__main__":
 
     try:
         opts, args = getopt.gnu_getopt(sys.argv[1:],
-                                       'Nh:p:u:P:d:a:m:Ms:Q:T:P:r:l:t:n:c:xOR:D:vgyH',
+                                       'Nh:p:u:P:d:a:m:Ms:Q:T:C:r:l:t:n:c:xOR:D:vgyH',
                                        [
             'native', 'host', 'port', 'user', 'password', 'dbname', 'replica', 'tbname',
-            'stable', 'stbname', 'query', 'numOfThreads', 'numOfProcesses',
+            'stable', 'stbname', 'query', 'threads', 'processes',
             'recPerReq', 'colsPerRecord', 'numOfTb', 'numOfRec', 'config',
             'insertOnly', 'outOfOrder', 'rateOOOO', 'deleteMethod',
             'verbose', 'debug', 'skipPrompt', 'help'
@@ -511,9 +518,9 @@ if __name__ == "__main__":
                   '\t-s, --stbname <stable prefix>     stable_prefix, STable prefix name. Default is \'st\'')
             print('\t-Q, --query <DEFAULT | command>   query, Execute query command. set \'DEFAULT\' means select * from each table')
             print(
-                '\t-T, --numOfThreads <number>        num_of_threads, The number of threads. Default is 1.')
+                  '\t-T, --threads <number>            num_of_threads, The number of threads. Default is 1.')
             print(
-                '\t-P, --numOfProcesses <number>      num_of_processes, The number of threads. Default is 1.')
+                  '\t-C, --processes <number>          num_of_processes, The number of threads. Default is 1.')
             print('\t-r, --batch <number>              num_of_records_per_req, The number of records per request. Default is 1000.')
             print(
                   '\t-t, --numOfTb <number>            num_of_tables, The number of tables. Default is 1.')
@@ -574,13 +581,13 @@ if __name__ == "__main__":
         if key in ['-Q', '--query']:
             queryCmd = str(value)
 
-        if key in ['-T', '--numOfThreads']:
+        if key in ['-T', '--threads']:
             threads = int(value)
             if threads < 1:
                 print("FATAL: number of threads must be larger than 0")
                 sys.exit(1)
 
-        if key in ['-P', '--numOfProcesses']:
+        if key in ['-C', '--processes']:
             processes = int(value)
             if processes < 1:
                 print("FATAL: number of processes must be larger than 0")
@@ -653,7 +660,6 @@ if __name__ == "__main__":
             print("Error: %s" % e.args[0])
             sys.exit(1)
 
-    if native:
         try:
             cursor = conn.cursor()
             print("cursor:%d %s" % (id(cursor), str(cursor.__class__)))
@@ -717,6 +723,7 @@ if __name__ == "__main__":
 
         sys.exit(0)
 
+    print("CBD LN755 %d" % numOfTb)
     if numOfTb > 0:
         create_tb()
         insert_data(processes)
@@ -739,11 +746,17 @@ if __name__ == "__main__":
                             host, port, user, password, "SELECT COUNT(*) FROM %s%d" %
                             (tbName, j))
 
-    if queryCmd != "":
+    if queryCmd != "DEFAULT":
         print("queryCmd: %s" % queryCmd)
+#        cursor.close() ##
+#        conn.close()  ## CBD
         query_data(queryCmd)
         sys.exit(0)
 
+    if native:
+        cursor.close()
+        conn.close()
+
     print("done")
     if measure:
         end_time = time.time()