未验证 提交 889019b2 编写于 作者: S sunpeng 提交者: GitHub

build: add python demo to ci (#19699)

* build: add python demo to ci

* build: fix python demo
上级 ac9cdf6c
module goexample
go 1.17
require github.com/taosdata/driver-go/v3 3.0
import pandas import pandas
from sqlalchemy import create_engine from sqlalchemy import create_engine, text
engine = create_engine("taos://root:taosdata@localhost:6030/power") engine = create_engine("taos://root:taosdata@localhost:6030/power")
df = pandas.read_sql("SELECT * FROM meters", engine) conn = engine.connect()
df = pandas.read_sql(text("SELECT * FROM power.meters"), conn)
conn.close()
# print index # print index
print(df.index) print(df.index)
......
import pandas import pandas
from sqlalchemy import create_engine from sqlalchemy import create_engine, text
engine = create_engine("taosrest://root:taosdata@localhost:6041") engine = create_engine("taosrest://root:taosdata@localhost:6041")
df: pandas.DataFrame = pandas.read_sql("SELECT * FROM power.meters", engine) conn = engine.connect()
df: pandas.DataFrame = pandas.read_sql(text("SELECT * FROM power.meters"), conn)
conn.close()
# print index # print index
print(df.index) print(df.index)
......
# ANCHOR: connect # ANCHOR: connect
from taosrest import connect, TaosRestConnection, TaosRestCursor from taosrest import connect, TaosRestConnection, TaosRestCursor
conn: TaosRestConnection = connect(url="http://localhost:6041", conn = connect(url="http://localhost:6041",
user="root", user="root",
password="taosdata", password="taosdata",
timeout=30) timeout=30)
...@@ -9,16 +9,17 @@ conn: TaosRestConnection = connect(url="http://localhost:6041", ...@@ -9,16 +9,17 @@ conn: TaosRestConnection = connect(url="http://localhost:6041",
# ANCHOR_END: connect # ANCHOR_END: connect
# ANCHOR: basic # ANCHOR: basic
# create STable # create STable
cursor: TaosRestCursor = conn.cursor() cursor = conn.cursor()
cursor.execute("DROP DATABASE IF EXISTS power") cursor.execute("DROP DATABASE IF EXISTS power")
cursor.execute("CREATE DATABASE power") cursor.execute("CREATE DATABASE power")
cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") cursor.execute(
"CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")
# insert data # insert data
cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)
power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)
power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000)
power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""")
print("inserted row count:", cursor.rowcount) print("inserted row count:", cursor.rowcount)
# query data # query data
...@@ -28,7 +29,7 @@ print("queried row count:", cursor.rowcount) ...@@ -28,7 +29,7 @@ print("queried row count:", cursor.rowcount)
# get column names from cursor # get column names from cursor
column_names = [meta[0] for meta in cursor.description] column_names = [meta[0] for meta in cursor.description]
# get rows # get rows
data: list[tuple] = cursor.fetchall() data = cursor.fetchall()
print(column_names) print(column_names)
for row in data: for row in data:
print(row) print(row)
......
...@@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test") ...@@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test")
# change database. same as execute "USE db" # change database. same as execute "USE db"
conn.select_db("test") conn.select_db("test")
conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m 24.4)") affected_row = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)")
print("affected_row", affected_row) print("affected_row", affected_row)
# output: # output:
# affected_row 3 # affected_row 3
...@@ -16,10 +16,10 @@ print("affected_row", affected_row) ...@@ -16,10 +16,10 @@ print("affected_row", affected_row)
# ANCHOR: query # ANCHOR: query
# Execute a sql and get its result set. It's useful for SELECT statement # Execute a sql and get its result set. It's useful for SELECT statement
result: taos.TaosResult = conn.query("SELECT * from weather") result = conn.query("SELECT * from weather")
# Get fields from result # Get fields from result
fields: taos.field.TaosFields = result.fields fields = result.fields
for field in fields: for field in fields:
print(field) # {name: ts, type: 9, bytes: 8} print(field) # {name: ts, type: 9, bytes: 8}
......
# install dependencies: # install dependencies:
# recommend python >= 3.8 # recommend python >= 3.8
# pip3 install faster-fifo
# #
import logging import logging
import math import math
import multiprocessing
import sys import sys
import time import time
import os import os
from multiprocessing import Process from multiprocessing import Process, Queue
from faster_fifo import Queue
from mockdatasource import MockDataSource from mockdatasource import MockDataSource
from queue import Empty from queue import Empty
from typing import List from typing import List
...@@ -22,8 +21,7 @@ TABLE_COUNT = 1000 ...@@ -22,8 +21,7 @@ TABLE_COUNT = 1000
QUEUE_SIZE = 1000000 QUEUE_SIZE = 1000000
MAX_BATCH_SIZE = 3000 MAX_BATCH_SIZE = 3000
read_processes = [] _DONE_MESSAGE = '__DONE__'
write_processes = []
def get_connection(): def get_connection():
...@@ -44,41 +42,64 @@ def get_connection(): ...@@ -44,41 +42,64 @@ def get_connection():
# ANCHOR: read # ANCHOR: read
def run_read_task(task_id: int, task_queues: List[Queue]): def run_read_task(task_id: int, task_queues: List[Queue], infinity):
table_count_per_task = TABLE_COUNT // READ_TASK_COUNT table_count_per_task = TABLE_COUNT // READ_TASK_COUNT
data_source = MockDataSource(f"tb{task_id}", table_count_per_task) data_source = MockDataSource(f"tb{task_id}", table_count_per_task, infinity)
try: try:
for batch in data_source: for batch in data_source:
if isinstance(batch, tuple):
batch = [batch]
for table_id, rows in batch: for table_id, rows in batch:
# hash data to different queue # hash data to different queue
i = table_id % len(task_queues) i = table_id % len(task_queues)
# block putting forever when the queue is full # block putting forever when the queue is full
task_queues[i].put_many(rows, block=True, timeout=-1) for row in rows:
task_queues[i].put(row)
if not infinity:
for queue in task_queues:
queue.put(_DONE_MESSAGE)
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
finally:
logging.info('read task over')
# ANCHOR_END: read # ANCHOR_END: read
# ANCHOR: write # ANCHOR: write
def run_write_task(task_id: int, queue: Queue): def run_write_task(task_id: int, queue: Queue, done_queue: Queue):
from sql_writer import SQLWriter from sql_writer import SQLWriter
log = logging.getLogger(f"WriteTask-{task_id}") log = logging.getLogger(f"WriteTask-{task_id}")
writer = SQLWriter(get_connection) writer = SQLWriter(get_connection)
lines = None lines = None
try: try:
while True: while True:
over = False
lines = []
for _ in range(MAX_BATCH_SIZE):
try: try:
# get as many as possible line = queue.get_nowait()
lines = queue.get_many(block=False, max_messages_to_get=MAX_BATCH_SIZE) if line == _DONE_MESSAGE:
writer.process_lines(lines) over = True
break
if line:
lines.append(line)
except Empty: except Empty:
time.sleep(0.01) time.sleep(0.1)
if len(lines) > 0:
writer.process_lines(lines)
if over:
done_queue.put(_DONE_MESSAGE)
break
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
except BaseException as e: except BaseException as e:
log.debug(f"lines={lines}") log.debug(f"lines={lines}")
raise e raise e
finally:
writer.close()
log.debug('write task over')
# ANCHOR_END: write # ANCHOR_END: write
...@@ -103,13 +124,11 @@ def set_global_config(): ...@@ -103,13 +124,11 @@ def set_global_config():
# ANCHOR: monitor # ANCHOR: monitor
def run_monitor_process(): def run_monitor_process(done_queue: Queue):
log = logging.getLogger("DataBaseMonitor") log = logging.getLogger("DataBaseMonitor")
conn = None
try:
conn = get_connection() conn = get_connection()
conn.execute("DROP DATABASE IF EXISTS test")
conn.execute("CREATE DATABASE test")
conn.execute("CREATE STABLE test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
"TAGS (location BINARY(64), groupId INT)")
def get_count(): def get_count():
res = conn.query("SELECT count(*) FROM test.meters") res = conn.query("SELECT count(*) FROM test.meters")
...@@ -118,32 +137,51 @@ def run_monitor_process(): ...@@ -118,32 +137,51 @@ def run_monitor_process():
last_count = 0 last_count = 0
while True: while True:
try:
done = done_queue.get_nowait()
if done == _DONE_MESSAGE:
break
except Empty:
pass
time.sleep(10) time.sleep(10)
count = get_count() count = get_count()
log.info(f"count={count} speed={(count - last_count) / 10}") log.info(f"count={count} speed={(count - last_count) / 10}")
last_count = count last_count = count
finally:
conn.close()
# ANCHOR_END: monitor # ANCHOR_END: monitor
# ANCHOR: main # ANCHOR: main
def main(): def main(infinity):
set_global_config() set_global_config()
logging.info(f"READ_TASK_COUNT={READ_TASK_COUNT}, WRITE_TASK_COUNT={WRITE_TASK_COUNT}, " logging.info(f"READ_TASK_COUNT={READ_TASK_COUNT}, WRITE_TASK_COUNT={WRITE_TASK_COUNT}, "
f"TABLE_COUNT={TABLE_COUNT}, QUEUE_SIZE={QUEUE_SIZE}, MAX_BATCH_SIZE={MAX_BATCH_SIZE}") f"TABLE_COUNT={TABLE_COUNT}, QUEUE_SIZE={QUEUE_SIZE}, MAX_BATCH_SIZE={MAX_BATCH_SIZE}")
monitor_process = Process(target=run_monitor_process) conn = get_connection()
conn.execute("DROP DATABASE IF EXISTS test")
conn.execute("CREATE DATABASE IF NOT EXISTS test")
conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
"TAGS (location BINARY(64), groupId INT)")
conn.close()
done_queue = Queue()
monitor_process = Process(target=run_monitor_process, args=(done_queue,))
monitor_process.start() monitor_process.start()
time.sleep(3) # waiting for database ready. logging.debug(f"monitor task started with pid {monitor_process.pid}")
task_queues: List[Queue] = [] task_queues: List[Queue] = []
write_processes = []
read_processes = []
# create task queues # create task queues
for i in range(WRITE_TASK_COUNT): for i in range(WRITE_TASK_COUNT):
queue = Queue(max_size_bytes=QUEUE_SIZE) queue = Queue()
task_queues.append(queue) task_queues.append(queue)
# create write processes # create write processes
for i in range(WRITE_TASK_COUNT): for i in range(WRITE_TASK_COUNT):
p = Process(target=run_write_task, args=(i, task_queues[i])) p = Process(target=run_write_task, args=(i, task_queues[i], done_queue))
p.start() p.start()
logging.debug(f"WriteTask-{i} started with pid {p.pid}") logging.debug(f"WriteTask-{i} started with pid {p.pid}")
write_processes.append(p) write_processes.append(p)
...@@ -151,13 +189,19 @@ def main(): ...@@ -151,13 +189,19 @@ def main():
# create read processes # create read processes
for i in range(READ_TASK_COUNT): for i in range(READ_TASK_COUNT):
queues = assign_queues(i, task_queues) queues = assign_queues(i, task_queues)
p = Process(target=run_read_task, args=(i, queues)) p = Process(target=run_read_task, args=(i, queues, infinity))
p.start() p.start()
logging.debug(f"ReadTask-{i} started with pid {p.pid}") logging.debug(f"ReadTask-{i} started with pid {p.pid}")
read_processes.append(p) read_processes.append(p)
try: try:
monitor_process.join() monitor_process.join()
for p in read_processes:
p.join()
for p in write_processes:
p.join()
time.sleep(1)
return
except KeyboardInterrupt: except KeyboardInterrupt:
monitor_process.terminate() monitor_process.terminate()
[p.terminate() for p in read_processes] [p.terminate() for p in read_processes]
...@@ -176,5 +220,6 @@ def assign_queues(read_task_id, task_queues): ...@@ -176,5 +220,6 @@ def assign_queues(read_task_id, task_queues):
if __name__ == '__main__': if __name__ == '__main__':
main() multiprocessing.set_start_method('spawn')
main(False)
# ANCHOR_END: main # ANCHOR_END: main
...@@ -26,7 +26,8 @@ class Consumer(object): ...@@ -26,7 +26,8 @@ class Consumer(object):
'bath_consume': True, 'bath_consume': True,
'batch_size': 1000, 'batch_size': 1000,
'async_model': True, 'async_model': True,
'workers': 10 'workers': 10,
'testing': False
} }
LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
...@@ -46,6 +47,7 @@ class Consumer(object): ...@@ -46,6 +47,7 @@ class Consumer(object):
def __init__(self, **configs): def __init__(self, **configs):
self.config: dict = self.DEFAULT_CONFIGS self.config: dict = self.DEFAULT_CONFIGS
self.config.update(configs) self.config.update(configs)
if not self.config.get('testing'):
self.consumer = KafkaConsumer( self.consumer = KafkaConsumer(
self.config.get('kafka_topic'), # topic self.config.get('kafka_topic'), # topic
bootstrap_servers=self.config.get('kafka_brokers'), bootstrap_servers=self.config.get('kafka_brokers'),
...@@ -60,7 +62,7 @@ class Consumer(object): ...@@ -60,7 +62,7 @@ class Consumer(object):
) )
if self.config.get('async_model'): if self.config.get('async_model'):
self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers')) self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers'))
self.tasks: list[Future] = [] self.tasks = []
# tags and table mapping # key: {location}_{groupId} value: # tags and table mapping # key: {location}_{groupId} value:
self.tag_table_mapping = {} self.tag_table_mapping = {}
i = 0 i = 0
...@@ -115,14 +117,14 @@ class Consumer(object): ...@@ -115,14 +117,14 @@ class Consumer(object):
if self.taos is not None: if self.taos is not None:
self.taos.close() self.taos.close()
def _run(self, f: Callable[[ConsumerRecord], bool]): def _run(self, f):
for message in self.consumer: for message in self.consumer:
if self.config.get('async_model'): if self.config.get('async_model'):
self.pool.submit(f(message)) self.pool.submit(f(message))
else: else:
f(message) f(message)
def _run_batch(self, f: Callable[[list[list[ConsumerRecord]]], None]): def _run_batch(self, f):
while True: while True:
messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size')) messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size'))
if messages: if messages:
...@@ -140,7 +142,7 @@ class Consumer(object): ...@@ -140,7 +142,7 @@ class Consumer(object):
logging.info('## insert sql %s', sql) logging.info('## insert sql %s', sql)
return self.taos.execute(sql=sql) == 1 return self.taos.execute(sql=sql) == 1
def _to_taos_batch(self, messages: list[list[ConsumerRecord]]): def _to_taos_batch(self, messages):
sql = self._build_sql_batch(messages=messages) sql = self._build_sql_batch(messages=messages)
if len(sql) == 0: # decode error, skip if len(sql) == 0: # decode error, skip
return return
...@@ -162,7 +164,7 @@ class Consumer(object): ...@@ -162,7 +164,7 @@ class Consumer(object):
table_name = self._get_table_name(location=location, group_id=group_id) table_name = self._get_table_name(location=location, group_id=group_id)
return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase) return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase)
def _build_sql_batch(self, messages: list[list[ConsumerRecord]]) -> str: def _build_sql_batch(self, messages) -> str:
sql_list = [] sql_list = []
for partition_messages in messages: for partition_messages in messages:
for message in partition_messages: for message in partition_messages:
...@@ -186,7 +188,55 @@ def _get_location_and_group(key: str) -> (str, int): ...@@ -186,7 +188,55 @@ def _get_location_and_group(key: str) -> (str, int):
return fields[0], fields[1] return fields[0], fields[1]
def test_to_taos(consumer: Consumer):
msg = {
'location': 'California.SanFrancisco',
'groupId': 1,
'ts': '2022-12-06 15:13:38.643',
'current': 3.41,
'voltage': 105,
'phase': 0.02027,
}
record = ConsumerRecord(checksum=None, headers=None, offset=1, key=None, value=json.dumps(msg), partition=1,
topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None)
assert consumer._to_taos(message=record)
def test_to_taos_batch(consumer: Consumer):
records = [
[
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'location': 'California.SanFrancisco',
'groupId': 1,
'ts': '2022-12-06 15:13:38.643',
'current': 3.41,
'voltage': 105,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'location': 'California.LosAngles',
'groupId': 2,
'ts': '2022-12-06 15:13:39.643',
'current': 3.41,
'voltage': 102,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
]
]
consumer._to_taos_batch(messages=records)
if __name__ == '__main__': if __name__ == '__main__':
consumer = Consumer(async_model=True) consumer = Consumer(async_model=True, testing=True)
# init env
consumer.init_env() consumer.init_env()
consumer.consume() # consumer.consume()
\ No newline at end of file # test build sql
# test build sql batch
test_to_taos(consumer)
test_to_taos_batch(consumer)
\ No newline at end of file
...@@ -10,13 +10,14 @@ class MockDataSource: ...@@ -10,13 +10,14 @@ class MockDataSource:
"9.4,118,0.141,California.SanFrancisco,4" "9.4,118,0.141,California.SanFrancisco,4"
] ]
def __init__(self, tb_name_prefix, table_count): def __init__(self, tb_name_prefix, table_count, infinity=True):
self.table_name_prefix = tb_name_prefix + "_" self.table_name_prefix = tb_name_prefix + "_"
self.table_count = table_count self.table_count = table_count
self.max_rows = 10000000 self.max_rows = 10000000
self.current_ts = round(time.time() * 1000) - self.max_rows * 100 self.current_ts = round(time.time() * 1000) - self.max_rows * 100
# [(tableId, tableName, values),] # [(tableId, tableName, values),]
self.data = self._init_data() self.data = self._init_data()
self.infinity = infinity
def _init_data(self): def _init_data(self):
lines = self.samples * (self.table_count // 5 + 1) lines = self.samples * (self.table_count // 5 + 1)
...@@ -28,6 +29,9 @@ class MockDataSource: ...@@ -28,6 +29,9 @@ class MockDataSource:
def __iter__(self): def __iter__(self):
self.row = 0 self.row = 0
if not self.infinity:
return iter(self._iter_data())
else:
return self return self
def __next__(self): def __next__(self):
...@@ -35,7 +39,9 @@ class MockDataSource: ...@@ -35,7 +39,9 @@ class MockDataSource:
next 1000 rows for each table. next 1000 rows for each table.
return: {tableId:[row,...]} return: {tableId:[row,...]}
""" """
# generate 1000 timestamps return self._iter_data()
def _iter_data(self):
ts = [] ts = []
for _ in range(1000): for _ in range(1000):
self.current_ts += 100 self.current_ts += 100
...@@ -47,3 +53,10 @@ class MockDataSource: ...@@ -47,3 +53,10 @@ class MockDataSource:
rows = [table_name + ',' + t + ',' + values for t in ts] rows = [table_name + ',' + t + ',' + values for t in ts]
result.append((table_id, rows)) result.append((table_id, rows))
return result return result
if __name__ == '__main__':
datasource = MockDataSource('t', 10, False)
for data in datasource:
print(data)
\ No newline at end of file
...@@ -10,6 +10,7 @@ class SQLWriter: ...@@ -10,6 +10,7 @@ class SQLWriter:
self._tb_tags = {} self._tb_tags = {}
self._conn = get_connection_func() self._conn = get_connection_func()
self._max_sql_length = self.get_max_sql_length() self._max_sql_length = self.get_max_sql_length()
self._conn.execute("create database if not exists test")
self._conn.execute("USE test") self._conn.execute("USE test")
def get_max_sql_length(self): def get_max_sql_length(self):
...@@ -20,7 +21,7 @@ class SQLWriter: ...@@ -20,7 +21,7 @@ class SQLWriter:
return int(r[1]) return int(r[1])
return 1024 * 1024 return 1024 * 1024
def process_lines(self, lines: str): def process_lines(self, lines: [str]):
""" """
:param lines: [[tbName,ts,current,voltage,phase,location,groupId]] :param lines: [[tbName,ts,current,voltage,phase,location,groupId]]
""" """
...@@ -60,6 +61,7 @@ class SQLWriter: ...@@ -60,6 +61,7 @@ class SQLWriter:
buf.append(q) buf.append(q)
sql_len += len(q) sql_len += len(q)
sql += " ".join(buf) sql += " ".join(buf)
self.create_tables()
self.execute_sql(sql) self.execute_sql(sql)
self._tb_values.clear() self._tb_values.clear()
...@@ -88,3 +90,23 @@ class SQLWriter: ...@@ -88,3 +90,23 @@ class SQLWriter:
except BaseException as e: except BaseException as e:
self.log.error("Execute SQL: %s", sql) self.log.error("Execute SQL: %s", sql)
raise e raise e
def close(self):
if self._conn:
self._conn.close()
if __name__ == '__main__':
def get_connection_func():
conn = taos.connect()
return conn
writer = SQLWriter(get_connection_func=get_connection_func)
writer.execute_sql(
"create stable if not exists meters (ts timestamp, current float, voltage int, phase float) "
"tags (location binary(64), groupId int)")
writer.execute_sql(
"INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) "
"VALUES ('2021-07-13 14:06:32.272', 10.2, 219, 0.32)")
\ No newline at end of file
from taos.tmq import Consumer
import taos import taos
from taos.tmq import *
conn = taos.connect()
print("init") def init_tmq_env(db, topic):
conn.execute("drop topic if exists topic_ctb_column") conn = taos.connect()
conn.execute("drop database if exists py_tmq") conn.execute("drop topic if exists {}".format(topic))
conn.execute("create database if not exists py_tmq vgroups 2") conn.execute("drop database if exists {}".format(db))
conn.select_db("py_tmq") conn.execute("create database if not exists {}".format(db))
conn.execute( conn.select_db(db)
"create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" conn.execute(
) "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))")
conn.execute("create table if not exists tb1 using stb1 tags(1)") conn.execute("create table if not exists tb1 using stb1 tags(1, 't1')")
conn.execute("create table if not exists tb2 using stb1 tags(2)") conn.execute("create table if not exists tb2 using stb1 tags(2, 't2')")
conn.execute("create table if not exists tb3 using stb1 tags(3)") conn.execute("create table if not exists tb3 using stb1 tags(3, 't3')")
conn.execute("create topic if not exists {} as select ts, c1, c2, c3 from stb1".format(topic))
print("create topic") conn.execute("insert into tb1 values (now, 1, 1.0, 'tmq test')")
conn.execute( conn.execute("insert into tb2 values (now, 2, 2.0, 'tmq test')")
"create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" conn.execute("insert into tb3 values (now, 3, 3.0, 'tmq test')")
)
print("build consumer") def cleanup(db, topic):
conf = TaosTmqConf() conn = taos.connect()
conf.set("group.id", "tg2") conn.execute("drop topic if exists {}".format(topic))
conf.set("td.connect.user", "root") conn.execute("drop database if exists {}".format(db))
conf.set("td.connect.pass", "taosdata")
conf.set("enable.auto.commit", "true")
if __name__ == '__main__':
init_tmq_env("tmq_test", "tmq_test_topic") # init env
def tmq_commit_cb_print(tmq, resp, offset, param=None): consumer = Consumer(
print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") {
"group.id": "tg2",
"td.connect.user": "root",
conf.set_auto_commit_cb(tmq_commit_cb_print, None) "td.connect.pass": "taosdata",
tmq = conf.new_consumer() "enable.auto.commit": "true",
}
print("build topic list") )
consumer.subscribe(["tmq_test_topic"])
topic_list = TaosTmqList()
topic_list.append("topic_ctb_column") try:
while True:
print("basic consume loop") res = consumer.poll(1)
tmq.subscribe(topic_list) if not res:
break
sub_list = tmq.subscription() err = res.error()
if err is not None:
print("subscribed topics: ", sub_list) raise err
val = res.value()
while 1:
res = tmq.poll(1000) for block in val:
if res: print(block.fetchall())
topic = res.get_topic_name() finally:
vg = res.get_vgroup_id() consumer.unsubscribe()
db = res.get_db_name() consumer.close()
print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") cleanup("tmq_test", "tmq_test_topic")
for row in res: \ No newline at end of file
print(row)
...@@ -44,4 +44,43 @@ taos -s "drop database test" ...@@ -44,4 +44,43 @@ taos -s "drop database test"
python3 json_protocol_example.py python3 json_protocol_example.py
# 10 # 10
# python3 subscribe_demo.py pip install SQLAlchemy
pip install pandas
taosBenchmark -y -d power -t 10 -n 10
python3 conn_native_pandas.py
python3 conn_rest_pandas.py
taos -s "drop database if exists power"
# 11
taos -s "create database if not exists test"
python3 connect_native_reference.py
# 12
python3 connect_rest_examples.py
# 13
python3 handle_exception.py
# 14
taosBenchmark -y -d power -t 2 -n 10
python3 rest_client_example.py
taos -s "drop database if exists power"
# 15
python3 result_set_examples.py
# 16
python3 tmq_example.py
# 17
python3 sql_writer.py
# 18
python3 mockdatasource.py
# 19
python3 fast_write_example.py
# 20
pip3 install kafka-python
python3 kafka_example.py
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册