Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 e5908c5f 编写于 作者: sangshuduo's avatar sangshuduo
浏览文件
操作

[TD-2771] feature: python taosdemo. multi process/thread working.

上级 11b1467d
隐藏空白更改
内联 并排
Showing with 139 addition and 101 deletion
tests/examples/python/taosdemo/taosdemo.py
浏览文件 @ e5908c5f
...@@ -21,7 +21,7 @@ import json ...@@ -21,7 +21,7 @@ import json
import random import random
import time import time
import datetime import datetime
from multiprocessing import Process, Pool, Lock from multiprocessing import Manager, Pool, Lock
from multipledispatch import dispatch from multipledispatch import dispatch
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
...@@ -176,7 +176,6 @@ def insert_data(processes: int): ...@@ -176,7 +176,6 @@ def insert_data(processes: int):
quotient, quotient,
remainder) remainder)
print("CBD LN210 processes:%d" % processes)
for i in range(processes): for i in range(processes):
begin = end begin = end
...@@ -274,67 +273,84 @@ def insert_func(process: int, thread: int): ...@@ -274,67 +273,84 @@ def insert_func(process: int, thread: int):
uuid = "%s" % uuid_int uuid = "%s" % uuid_int
v_print("uuid is: %s", uuid) v_print("uuid is: %s", uuid)
# establish connection if native
if native:
v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir)
try:
conn = taos.connect(
host=host,
user=user,
password=password,
config=configDir)
print("conn: %s" % str(conn.__class__))
except Exception as e:
print("Error: %s" % e.args[0])
sys.exit(1)
try:
cursor = conn.cursor()
print("cursor:%d %s" % (id(cursor), str(cursor.__class__)))
except Exception as e:
print("Error: %s" % e.args[0])
sys.exit(1)
v_print("numOfRec %d:", numOfRec) v_print("numOfRec %d:", numOfRec)
if numOfRec > 0:
row = 0
while row < numOfRec:
v_print("row: %d", row)
sqlCmd = ['INSERT INTO ']
try:
sqlCmd.append(
"%s.%s%d " % (current_db, tbName, thread))
if (numOfStb > 0 and autosubtable):
sqlCmd.append("USING %s.%s%d TAGS('%s') " %
(current_db, stbName, numOfStb - 1, uuid))
start_time = datetime.datetime(
2021, 1, 25) + datetime.timedelta(seconds=row)
sqlCmd.append("VALUES ")
for batchIter in range(0, batch):
sqlCmd.append("(now, %f) " %
(
# start_time +
# datetime.timedelta(
# milliseconds=batchIter),
random.random()))
row = row + 1
if row >= numOfRec:
v_print("BREAK, row: %d numOfRec:%d", row, numOfRec)
break
except Exception as e: row = 0
print("Error: %s" % e.args[0]) while row < numOfRec:
v_print("row: %d", row)
sqlCmd = ['INSERT INTO ']
try:
sqlCmd.append(
"%s.%s%d " % (current_db, tbName, thread))
if (numOfStb > 0 and autosubtable):
sqlCmd.append("USING %s.%s%d TAGS('%s') " %
(current_db, stbName, numOfStb - 1, uuid))
start_time = datetime.datetime(
2021, 1, 25) + datetime.timedelta(seconds=row)
sqlCmd.append("VALUES ")
for batchIter in range(0, batch):
sqlCmd.append("('%s', %f) " %
(
start_time +
datetime.timedelta(
milliseconds=batchIter),
random.random()))
row = row + 1
if row >= numOfRec:
v_print("BREAK, row: %d numOfRec:%d", row, numOfRec)
break
cmd = ' '.join(sqlCmd) except Exception as e:
print("Error: %s" % e.args[0])
print("CBD: LN313") cmd = ' '.join(sqlCmd)
if measure:
exec_start_time = datetime.datetime.now()
print("CBD: LN316 native: %d" % native) if measure:
if native: exec_start_time = datetime.datetime.now()
print("CBD: LN319: %s" % cmd)
print("conn: %s" % str(conn.__class__))
print("CBD: LN320 cursor:%d %s" % (id(cursor), str(cursor.__class__)))
# cursor.execute("SHOW DATABASES" )
affectedRows = cursor.execute(cmd)
print("CBD: LN323 affectedRows:%d" % affectedRows)
else:
restful_execute(
host, port, user, password, cmd)
print("CBD: LN327") if native:
if measure: affectedRows = cursor.execute(cmd)
exec_end_time = datetime.datetime.now() else:
exec_delta = exec_end_time - exec_start_time restful_execute(
print( host, port, user, password, cmd)
"%s, %d" %
(time.strftime('%X'),
exec_delta.microseconds))
v_print("cmd: %s, length:%d", cmd, len(cmd)) if measure:
exec_end_time = datetime.datetime.now()
exec_delta = exec_end_time - exec_start_time
print(
"%s, %d" %
(time.strftime('%X'),
exec_delta.microseconds))
v_print("cmd: %s, length:%d", cmd, len(cmd))
if native:
cursor.close()
conn.close()
def create_tb_using_stb(): def create_tb_using_stb():
...@@ -367,11 +383,10 @@ def create_tb(): ...@@ -367,11 +383,10 @@ def create_tb():
(tbName, j)) (tbName, j))
def insert_data_process(i_lock, i: int, begin: int, end: int): def insert_data_process(lock, i: int, begin: int, end: int):
print("CBD LN371 insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) lock.acquire()
time.sleep(0.01)
tasks = end - begin tasks = end - begin
i_lock.aquire() v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks)
if (threads < (end - begin)): if (threads < (end - begin)):
for j in range(begin, end, threads): for j in range(begin, end, threads):
...@@ -395,7 +410,9 @@ def insert_data_process(i_lock, i: int, begin: int, end: int): ...@@ -395,7 +410,9 @@ def insert_data_process(i_lock, i: int, begin: int, end: int):
begin, begin,
end)] end)]
wait(workers, return_when=ALL_COMPLETED) wait(workers, return_when=ALL_COMPLETED)
i_lock.release()
lock.release()
def query_db(i): def query_db(i):
if native: if native:
...@@ -624,6 +641,10 @@ if __name__ == "__main__": ...@@ -624,6 +641,10 @@ if __name__ == "__main__":
if key in ['-n', '--numOfRec']: if key in ['-n', '--numOfRec']:
numOfRec = int(value) numOfRec = int(value)
v_print("numOfRec is %d", numOfRec) v_print("numOfRec is %d", numOfRec)
if numOfRec < 1:
print("FATAL: number of records must be larger than 0")
sys.exit(1)
if key in ['-c', '--config']: if key in ['-c', '--config']:
configDir = value configDir = value
...@@ -665,6 +686,7 @@ if __name__ == "__main__": ...@@ -665,6 +686,7 @@ if __name__ == "__main__":
if not skipPrompt: if not skipPrompt:
input("Press any key to continue..") input("Press any key to continue..")
# establish connection first if native
if native: if native:
v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir)
try: try:
...@@ -685,8 +707,7 @@ if __name__ == "__main__": ...@@ -685,8 +707,7 @@ if __name__ == "__main__":
print("Error: %s" % e.args[0]) print("Error: %s" % e.args[0])
sys.exit(1) sys.exit(1)
# drop data only if delete method be set
if deleteMethod > 0: if deleteMethod > 0:
if deleteMethod == 1: if deleteMethod == 1:
drop_tables() drop_tables()
...@@ -700,69 +721,86 @@ if __name__ == "__main__": ...@@ -700,69 +721,86 @@ if __name__ == "__main__":
sys.exit(0) sys.exit(0)
# create databases # create databases
if (insertOnly == False): drop_databases()
drop_databases()
create_databases() create_databases()
if measure:
start_time = time.time()
# use last database # use last database
current_db = "%s%d" % (dbName, (numOfDb - 1)) current_db = "%s%d" % (dbName, (numOfDb - 1))
use_database() use_database()
if measure:
start_time_begin = time.time()
if numOfStb > 0: if numOfStb > 0:
create_stb() create_stb()
if (autosubtable == False): if (autosubtable == False):
create_tb_using_stb() create_tb_using_stb()
else:
create_tb()
insert_data(processes) if measure:
end_time = time.time()
print(
"Total time consumed {} seconds for create table.".format(
(end_time - start_time_begin)))
if verbose: if native:
for i in range(0, numOfDb): cursor.close()
for j in range(0, numOfStb): conn.close()
if native:
cursor.execute(
"SELECT COUNT(*) FROM %s%d.%s%d" %
(dbName, i, stbName, j,))
else:
restful_execute(
host, port, user, password, "SELECT COUNT(*) FROM %s%d.%s%d" %
(dbName, i, stbName, j,))
print("done") # start insert data
if measure:
start_time = time.time()
if measure: manager = Manager()
end_time = time.time() lock = manager.Lock()
print( pool = Pool(processes)
"Total time consumed {} seconds.".format(
(end_time - start_time)))
sys.exit(0) begin = 0
end = 0
print("CBD LN755 %d" % numOfTb) quotient = numOfTb // processes
if numOfTb > 0: if quotient < 1:
create_tb() processes = numOfTb
insert_data(processes) quotient = 1
if debug: remainder = numOfTb % processes
for i in range(0, numOfDb): v_print(
query_db(i) "num of tables: %d, quotient: %d, remainder: %d",
numOfTb,
quotient,
remainder)
for i in range(processes):
begin = end
if i < remainder:
end = begin + quotient + 1
else:
end = begin + quotient
pool.apply_async(insert_data_process, args=(lock, i, begin, end,))
# pool.apply_async(text, args=(lock, i, begin, end,))
pool.close()
pool.join()
time.sleep(1)
if measure:
end_time = time.time()
print(
"Total time consumed {} seconds for insert data.".format(
(end_time - start_time)))
# query data
if queryCmd != "NO": if queryCmd != "NO":
print("queryCmd: %s" % queryCmd) print("queryCmd: %s" % queryCmd)
query_data(queryCmd) query_data(queryCmd)
sys.exit(0)
if native:
cursor.close()
conn.close()
if measure: if measure:
end_time = time.time() end_time = time.time()
print( print(
"Total time consumed {} seconds.".format( "Total time consumed {} seconds.".format(
(end_time - start_time))) (end_time - start_time_begin)))
print("done") print("done")
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或