提交 26b9a07e 编写于 作者: C Cary Xu

other: merge 3.0

...@@ -218,12 +218,12 @@ def pre_test_win(){ ...@@ -218,12 +218,12 @@ def pre_test_win(){
if (env.CHANGE_URL =~ /\/TDengine\//) { if (env.CHANGE_URL =~ /\/TDengine\//) {
bat ''' bat '''
cd %WIN_INTERNAL_ROOT% cd %WIN_INTERNAL_ROOT%
git pull git pull origin ''' + env.CHANGE_TARGET + '''
''' '''
bat ''' bat '''
cd %WIN_COMMUNITY_ROOT% cd %WIN_COMMUNITY_ROOT%
git remote prune origin git remote prune origin
git pull git pull origin ''' + env.CHANGE_TARGET + '''
''' '''
bat ''' bat '''
cd %WIN_COMMUNITY_ROOT% cd %WIN_COMMUNITY_ROOT%
...@@ -236,7 +236,7 @@ def pre_test_win(){ ...@@ -236,7 +236,7 @@ def pre_test_win(){
} else if (env.CHANGE_URL =~ /\/TDinternal\//) { } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
bat ''' bat '''
cd %WIN_INTERNAL_ROOT% cd %WIN_INTERNAL_ROOT%
git pull git pull origin ''' + env.CHANGE_TARGET + '''
''' '''
bat ''' bat '''
cd %WIN_INTERNAL_ROOT% cd %WIN_INTERNAL_ROOT%
......
...@@ -15,19 +15,19 @@ ...@@ -15,19 +15,19 @@
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
English | [简体中文](README-CN.md) | [Lean more about TSDB](https://tdengine.com/tsdb) English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengine.com/tsdb/)
# What is TDengine? # What is TDengine?
TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages: TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages:
- **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. - **[High Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression.
- **[Simplified Solution](https://tdengine.com/tdengine/simplified-time-series-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly. - **[Simplified Solution](https://tdengine.com/tdengine/simplified-time-series-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly.
- **[Cloud Native](https://tdengine.com/tdengine/cloud-native-time-series-database/)**: Through native distributed design, sharding and partitioning, separation of compute and storage, RAFT, support for kubernetes deployment and full observability, TDengine is a cloud native Time-Series Database and can be deployed on public, private or hybrid clouds. - **[Cloud Native](https://tdengine.com/tdengine/cloud-native-time-series-database/)**: Through native distributed design, sharding and partitioning, separation of compute and storage, RAFT, support for kubernetes deployment and full observability, TDengine is a cloud native Time-Series Database and can be deployed on public, private or hybrid clouds.
- **[Ease of Use](https://docs.tdengine.com/get-started/docker/)**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access. - **[Ease of Use](https://tdengine.com/tdengine/easy-time-series-data-platform/)**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access.
- **[Easy Data Analytics](https://tdengine.com/tdengine/time-series-data-analytics-made-easy/)**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way. - **[Easy Data Analytics](https://tdengine.com/tdengine/time-series-data-analytics-made-easy/)**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way.
...@@ -232,9 +232,9 @@ After building successfully, TDengine can be installed by ...@@ -232,9 +232,9 @@ After building successfully, TDengine can be installed by
sudo make install sudo make install
``` ```
Users can find more information about directories installed on the system in the [directory and files](https://docs.taosdata.com/reference/directory/) section. Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section.
Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.taosdata.com/get-started/package/) for it. Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it.
To start the service after installation, in a terminal, use: To start the service after installation, in a terminal, use:
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.0.1.1") SET(TD_VER_NUMBER "3.0.1.2")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 509ec72 GIT_TAG 8207c74
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# taosws-rs # taosws-rs
ExternalProject_Add(taosws-rs ExternalProject_Add(taosws-rs
GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
GIT_TAG e771403 GIT_TAG 1bdfca3
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE
......
此差异已折叠。
...@@ -68,7 +68,7 @@ A query can be performed on some or all columns. Data and tag columns can all be ...@@ -68,7 +68,7 @@ A query can be performed on some or all columns. Data and tag columns can all be
### Wildcards ### Wildcards
You can use an asterisk (\*) as a wildcard character to indicate all columns. For standard tables, the asterisk indicates only data columns. For supertables and subtables, tag columns are also included. You can use an asterisk (\*) as a wildcard character to indicate all columns. For normal tables or sub-tables, the asterisk indicates only data columns. For supertables, tag columns are also included when using asterisk (\*).
```sql ```sql
SELECT * FROM d1001; SELECT * FROM d1001;
......
...@@ -6,6 +6,9 @@ description: TDengine release history, Release Notes and download links. ...@@ -6,6 +6,9 @@ description: TDengine release history, Release Notes and download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" />
## 3.0.1.1 ## 3.0.1.1
......
...@@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links. ...@@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links.
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.2
<Release type="tools" version="2.2.2" />
## 2.2.0 ## 2.2.0
<Release type="tools" version="2.2.0" /> <Release type="tools" version="2.2.0" />
......
bin bin
obj obj
.vs .vs
*.sln *.sln
\ No newline at end of file wsConnect/obj
wsInsert/obj
wsQuery/obj
wsStmt/obj
wsConnect/bin
wsInsert/bin
wsQuery/bin
wsStmt/bin
\ No newline at end of file
using System;
using TDengineWS.Impl;
namespace Examples
{
public class WSConnExample
{
static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
{
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
}
else
{
Console.WriteLine("Establish connect success.");
}
// close connection.
LibTaosWS.WSClose(wsConn);
}
}
}
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.1" />
</ItemGroup>
</Project>
using System;
using TDengineWS.Impl;
namespace Examples
{
public class WSInsertExample
{
static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
// Assert if connection is validate
if (wsConn == IntPtr.Zero)
{
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
}
else
{
Console.WriteLine("Establish connect success.");
}
string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);";
string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" +
"test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" +
"test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
"test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable);
ValidInsert("create table", wsRes);
LibTaosWS.WSFreeResult(wsRes);
wsRes = LibTaosWS.WSQuery(wsConn, insert);
ValidInsert("insert data", wsRes);
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
}
static void ValidInsert(string desc, IntPtr wsRes)
{
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
}
else
{
Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes),LibTaosWS.WSAffectRows(wsRes));
}
}
}
}
// Establish connect success.
// create table success affect 0 rows, cost 3717542 nanoseconds
// insert data success affect 8 rows, cost 2613637 nanoseconds
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.1" />
</ItemGroup>
</Project>
using System;
using TDengineWS.Impl;
using System.Collections.Generic;
using TDengineDriver;
namespace Examples
{
public class WSQueryExample
{
static void Main(string[] args)
{
string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
{
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
}
else
{
Console.WriteLine("Establish connect success.");
}
string select = "select * from test.meters";
// optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
// Assert if query execute success.
int code = LibTaosWS.WSErrorNo(wsRes);
if (code != 0)
{
throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
}
// get meta data
List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
// get retrieved data
List<object> dataSet = LibTaosWS.WSGetData(wsRes);
// do something with result.
foreach (var meta in metas)
{
Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
}
Console.WriteLine("");
for (int i = 0; i < dataSet.Count;)
{
for (int j = 0; j < metas.Count; j++)
{
Console.Write("{0}\t|\t", dataSet[i]);
i++;
}
Console.WriteLine("");
}
// Free result after use.
LibTaosWS.WSFreeResult(wsRes);
// close connection.
LibTaosWS.WSClose(wsConn);
}
}
}
// Establish connect success.
// ts TIMESTAMP(8) | current FLOAT(4) | voltage INT(4) | phase FLOAT(4) | location BINARY(64) | groupid INT(4) |
// 1538548685000 | 10.8 | 223 | 0.29 | California.LosAngeles | 3 |
// 1538548686500 | 11.5 | 221 | 0.35 | California.LosAngeles | 3 |
// 1538548685500 | 11.8 | 221 | 0.28 | California.LosAngeles | 2 |
// 1538548696600 | 13.4 | 223 | 0.29 | California.LosAngeles | 2 |
// 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 |
// 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 |
// 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 |
// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 |
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.1" />
</ItemGroup>
</Project>
using System;
using TDengineWS.Impl;
using TDengineDriver;
using System.Runtime.InteropServices;
namespace Examples
{
public class WSStmtExample
{
static void Main(string[] args)
{
const string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
const string table = "meters";
const string database = "test";
const string childTable = "d1005";
string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
const int numOfTags = 2;
const int numOfColumns = 4;
// Establish connection
IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
if (wsConn == IntPtr.Zero)
{
throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
}
else
{
Console.WriteLine("Establish connect success...");
}
// init stmt
IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn);
if (wsStmt != IntPtr.Zero)
{
int code = LibTaosWS.WSStmtPrepare(wsStmt, insert);
ValidStmtStep(code, wsStmt, "WSStmtPrepare");
TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) };
code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags);
ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags");
TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4];
data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 });
data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F });
data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
code = LibTaosWS.WSStmtAddBatch(wsStmt);
ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
ValidStmtStep(code, wsStmt, "WSStmtExecute");
Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
Marshal.FreeHGlobal(stmtAffectRowPtr);
LibTaosWS.WSStmtClose(wsStmt);
// Free unmanaged memory
WSMultiBind.WSFreeTaosBind(wsTags);
WSMultiBind.WSFreeTaosBind(data);
//check result with SQL "SELECT * FROM test.d1005;"
}
else
{
throw new Exception("Init STMT failed...");
}
// close connection.
LibTaosWS.WSClose(wsConn);
}
static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
{
if (code != 0)
{
throw new Exception($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
}
else
{
Console.WriteLine("{0} success...", desc);
}
}
}
}
// WSStmtPrepare success...
// WSStmtSetTbnameTags success...
// WSStmtBindParamBatch success...
// WSStmtAddBatch success...
// WSStmtExecute success...
// WS STMT insert 5 rows...
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TDengine.Connector" Version="3.0.1" />
</ItemGroup>
</Project>
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
{{#include docs/examples/csharp/ConnectExample.cs}} {{#include docs/examples/csharp/ConnectExample.cs}}
``` ```
:::info ```csharp title="WebSocket 连接"
C# 连接器目前只支持原生连接。 {{#include docs/examples/csharp/wsConnect/Program.cs}}
```
:::
...@@ -17,7 +17,7 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" ...@@ -17,7 +17,7 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx"
`TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。
`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式,用户可以参考 [REST API](../rest-api/) 文档自行编写 `TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、数据订阅、schemaless 数据写入、参数绑定接口数据写入等功能。 `TDengine.Connector` 还支持 WebSocket,通过 DSN 建立 WebSocket 连接,提供数据写入、查询、参数绑定接口数据写入等功能
本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。
...@@ -35,12 +35,29 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" ...@@ -35,12 +35,29 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx"
## 支持的功能特性 ## 支持的功能特性
<Tabs defaultValue="native">
<TabItem value="native" label="原生连接">
1. 连接管理 1. 连接管理
2. 普通查询 2. 普通查询
3. 连续查询 3. 连续查询
4. 参数绑定 4. 参数绑定
5. 订阅功能 5. 数据订阅(TMQ)
6. Schemaless 6. Schemaless
</TabItem>
<TabItem value="rest" label="WebSocket 连接">
1. 连接管理
2. 普通查询
3. 连续查询
4. 参数绑定
</TabItem>
</Tabs>
## 安装步骤 ## 安装步骤
...@@ -79,7 +96,13 @@ dotnet add exmaple.csproj reference src/TDengine.csproj ...@@ -79,7 +96,13 @@ dotnet add exmaple.csproj reference src/TDengine.csproj
## 建立连接 ## 建立连接
``` C# <Tabs defaultValue="native">
<TabItem value="native" label="原生连接">
使用 host、username、password、port 等信息建立连接。
``` csharp
using TDengineDriver; using TDengineDriver;
namespace TDengineExample namespace TDengineExample
...@@ -109,17 +132,63 @@ namespace TDengineExample ...@@ -109,17 +132,63 @@ namespace TDengineExample
} }
} }
} }
```
</TabItem>
<TabItem value="rest" label="WebSocket 连接">
使用 DSN 建立 WebSocket 连接 DSN 连接。 描述字符串基本结构如下:
```text
[<protocol>]://[[<username>:<password>@]<host>:<port>][/<database>][?<p1>=<v1>[&<p2>=<v2>]]
|------------|---|-----------|-----------|------|------|------------|-----------------------|
| protocol | | username | password | host | port | database | params |
```
各部分意义见下表:
* **protocol**: 显示指定以何种方式建立连接,例如:`ws://localhost:6041` 指定以 Websocket 方式建立连接(支持http/ws)。
* **username/password**: 用于创建连接的用户名及密码(默认`root/taosdata`)。
* **host/port**: 指定创建连接的服务器及端口,WebSocket 连接默认为 `localhost:6041` 。
* **database**: 指定默认连接的数据库名。
* **params**:其他可选参数。
``` csharp
{{#include docs/examples/csharp/wsConnect/Program.cs}}
``` ```
</TabItem>
</Tabs>
## 使用示例 ## 使用示例
### 写入数据 ### 写入数据
#### SQL 写入 #### SQL 写入
<Tabs defaultValue="native">
<TabItem value="native" label="原生连接">
<CSInsert /> <CSInsert />
</TabItem>
<TabItem value="rest" label="WebSocket 连接">
```csharp
{{#include docs/examples/csharp/wsInsert/Program.cs}}
```
</TabItem>
</Tabs>
#### InfluxDB 行协议写入 #### InfluxDB 行协议写入
<CSInfluxLine /> <CSInfluxLine />
...@@ -132,12 +201,50 @@ namespace TDengineExample ...@@ -132,12 +201,50 @@ namespace TDengineExample
<CSOpenTSDBJson /> <CSOpenTSDBJson />
#### 参数绑定
<Tabs defaultValue="native">
<TabItem value="native" label="原生连接">
``` csharp
{{#include docs/examples/csharp/StmtInsertExample.cs}}
```
</TabItem>
<TabItem value="rest" label="WebSocket 连接">
```csharp
{{#include docs/examples/csharp/wsStmt/Program.cs}}
```
</TabItem>
</Tabs>
### 查询数据 ### 查询数据
#### 同步查询 #### 同步查询
<Tabs defaultValue="native">
<TabItem value="native" label="原生连接">
<CSQuery /> <CSQuery />
</TabItem>
<TabItem value="rest" label="WebSocket 连接">
```csharp
{{#include docs/examples/csharp/wsQuery/Program.cs}}
```
</TabItem>
</Tabs>
#### 异步查询 #### 异步查询
<CSAsyncQuery /> <CSAsyncQuery />
...@@ -151,12 +258,15 @@ namespace TDengineExample ...@@ -151,12 +258,15 @@ namespace TDengineExample
| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | | [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 |
| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | | [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 |
| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | | [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 |
| [TMQ](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | | [数据订阅(TMQ)](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 |
| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 |
| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 |
## 重要更新记录 ## 重要更新记录
| TDengine.Connector | 说明 | | TDengine.Connector | 说明 |
|--------------------|--------------------------------| |--------------------|--------------------------------|
| 3.0.1 | 支持 WebSocket 和 Cloud,查询,插入,参数绑定。 |
| 3.0.0 | 支持 TDengine 3.0.0.0,不兼容 2.x。新增接口TDengine.Impl.GetData(),解析查询结果。 | | 3.0.0 | 支持 TDengine 3.0.0.0,不兼容 2.x。新增接口TDengine.Impl.GetData(),解析查询结果。 |
| 1.0.7 | 修复 TDengine.Query()内存泄露。 | | 1.0.7 | 修复 TDengine.Query()内存泄露。 |
| 1.0.6 | 修复 schemaless 在 1.0.4 和 1.0.5 中失效 bug。 | | 1.0.6 | 修复 schemaless 在 1.0.4 和 1.0.5 中失效 bug。 |
......
--- ---
sidebar_label: 数据类型 sidebar_label: 数据类型
title: 数据类型 title: 数据类型
description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类型等" description: 'TDengine 支持的数据类型: 时间戳、浮点型、JSON 类型等'
--- ---
## 时间戳 ## 时间戳
...@@ -9,64 +9,65 @@ description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类 ...@@ -9,64 +9,65 @@ description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类
使用 TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则: 使用 TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则:
- 时间格式为 `YYYY-MM-DD HH:mm:ss.MS`,默认时间分辨率为毫秒。比如:`2017-08-12 18:25:58.128` - 时间格式为 `YYYY-MM-DD HH:mm:ss.MS`,默认时间分辨率为毫秒。比如:`2017-08-12 18:25:58.128`
- 内部函数 now 是客户端的当前时间 - 内部函数 NOW 是客户端的当前时间
- 插入记录时,如果时间戳为 now,插入数据时使用提交这条记录的客户端的当前时间 - 插入记录时,如果时间戳为 NOW,插入数据时使用提交这条记录的客户端的当前时间
- Epoch Time:时间戳也可以是一个长整数,表示从 UTC 时间 1970-01-01 00:00:00 开始的毫秒数。相应地,如果所在 Database 的时间精度设置为“微秒”,则长整型格式的时间戳含义也就对应于从 UTC 时间 1970-01-01 00:00:00 开始的微秒数;纳秒精度逻辑类似 - Epoch Time:时间戳也可以是一个长整数,表示从 UTC 时间 1970-01-01 00:00:00 开始的毫秒数。相应地,如果所在 Database 的时间精度设置为“微秒”,则长整型格式的时间戳含义也就对应于从 UTC 时间 1970-01-01 00:00:00 开始的微秒数;纳秒精度逻辑相同
- 时间可以加减,比如 now-2h,表明查询时刻向前推 2 个小时(最近 2 小时)。数字后面的时间单位可以是 b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`,表示查询两周前整整一周的数据。在指定降采样操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n (自然月) 和 y (自然年) - 时间可以加减,比如 NOW-2h,表明查询时刻向前推 2 个小时(最近 2 小时)。数字后面的时间单位可以是 b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `SELECT * FROM t1 WHERE ts > NOW-2w AND ts <= NOW-1w`,表示查询两周前整整一周的数据。在指定降采样操作(Down Sampling)的时间窗口(Interval)时,时间单位还可以使用 n(自然月)和 y(自然年)
TDengine 缺省的时间戳精度是毫秒,但通过在 `CREATE DATABASE` 时传递的 PRECISION 参数也可以支持微秒和纳秒。 TDengine 缺省的时间戳精度是毫秒,但通过在 `CREATE DATABASE` 时传递的 `PRECISION` 参数也可以支持微秒和纳秒。
```sql ```sql
CREATE DATABASE db_name PRECISION 'ns'; CREATE DATABASE db_name PRECISION 'ns';
``` ```
## 数据类型 ## 数据类型
在 TDengine 中,普通表的数据模型中可使用以下数据类型。 在 TDengine 中,普通表的数据模型中可使用以下数据类型。
| # | **类型** | **Bytes** | **说明** | | # | **类型** | **Bytes** | **说明** |
| --- | :-------: | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | --- | :---------------: | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒和纳秒,详细说明见上节。 | | 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒和纳秒,详细说明见上节。 |
| 2 | INT | 4 | 整型,范围 [-2^31, 2^31-1] | | 2 | INT | 4 | 整型,范围 [-2^31, 2^31-1] |
| 3 | INT UNSIGNED| 4| 无符号整数,[0, 2^32-1] | 3 | INT UNSIGNED | 4 | 无符号整数,[0, 2^32-1] |
| 4 | BIGINT | 8 | 长整型,范围 [-2^63, 2^63-1] | | 4 | BIGINT | 8 | 长整型,范围 [-2^63, 2^63-1] |
| 5 | BIGINT UNSIGNED | 8 | 长整型,范围 [0, 2^64-1] | | 5 | BIGINT UNSIGNED | 8 | 长整型,范围 [0, 2^64-1] |
| 6 | FLOAT | 4 | 浮点型,有效位数 6-7,范围 [-3.4E38, 3.4E38] | | 6 | FLOAT | 4 | 浮点型,有效位数 6-7,范围 [-3.4E38, 3.4E38] |
| 7 | DOUBLE | 8 | 双精度浮点型,有效位数 15-16,范围 [-1.7E308, 1.7E308] | | 7 | DOUBLE | 8 | 双精度浮点型,有效位数 15-16,范围 [-1.7E308, 1.7E308] |
| 8 | BINARY | 自定义 | 记录单字节字符串,建议只用于处理 ASCII 可见字符,中文等多字节字符需使用 nchar。 | | 8 | BINARY | 自定义 | 记录单字节字符串,建议只用于处理 ASCII 可见字符,中文等多字节字符需使用 NCHAR |
| 9 | SMALLINT | 2 | 短整型, 范围 [-32768, 32767] | | 9 | SMALLINT | 2 | 短整型, 范围 [-32768, 32767] |
| 10 | SMALLINT UNSIGNED | 2| 无符号短整型,范围 [0, 65535] | | 10 | SMALLINT UNSIGNED | 2 | 无符号短整型,范围 [0, 65535] |
| 11 | TINYINT | 1 | 单字节整型,范围 [-128, 127] | | 11 | TINYINT | 1 | 单字节整型,范围 [-128, 127] |
| 12 | TINYINT UNSIGNED | 1 | 无符号单字节整型,范围 [0, 255] | | 12 | TINYINT UNSIGNED | 1 | 无符号单字节整型,范围 [0, 255] |
| 13 | BOOL | 1 | 布尔型,{true, false} | | 13 | BOOL | 1 | 布尔型,{true, false} |
| 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串,如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用,字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小,类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符,会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度,将会报错。 | | 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串,如中文字符。每个 NCHAR 字符占用 4 字节的存储空间。字符串两端使用单引号引用,字符串内的单引号需用转义字符 `\'`。NCHAR 使用时须指定字符串大小,类型为 NCHAR(10) 的列表示此列的字符串最多存储 10 个 NCHAR 字符,会固定占用 40 字节的空间。如果用户字符串长度超出声明长度,将会报错。 |
| 15 | JSON | | json 数据类型, 只有 tag 可以是 json 格式 | | 15 | JSON | | JSON 数据类型, 只有 Tag 可以是 JSON 格式 |
| 16 | VARCHAR | 自定义 | BINARY类型的别名 | | 16 | VARCHAR | 自定义 | BINARY 类型的别名 |
:::note :::note
- TDengine 对 SQL 语句中的英文字符不区分大小写,自动转化为小写执行。因此用户大小写敏感的字符串及密码,需要使用单引号将字符串引起来。
- 虽然 BINARY 类型在底层存储上支持字节型的二进制字符,但不同编程语言对二进制数据的处理方式并不保证一致,因此建议在 BINARY 类型中只存储 ASCII 可见字符,而避免存储不可见字符。多字节的数据,例如中文字符,则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符,虽然有时也能正常读写,但并不带有字符集信息,很容易出现数据乱码甚至数据损坏等情况。 - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符,但不同编程语言对二进制数据的处理方式并不保证一致,因此建议在 BINARY 类型中只存储 ASCII 可见字符,而避免存储不可见字符。多字节的数据,例如中文字符,则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符,虽然有时也能正常读写,但并不带有字符集信息,很容易出现数据乱码甚至数据损坏等情况。
- BINARY 类型理论上最长可以有 16374 字节。binary 仅支持字符串输入,字符串两端需使用单引号引用。使用时须指定大小,如 binary(20) 定义了最长为 20 个单字节字符的字符串,每个字符占 1 byte 的存储空间,总共固定占用 20 bytes 的空间,此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示,即 `\’` - BINARY 类型理论上最长可以有 16,374 字节。BINARY 仅支持字符串输入,字符串两端需使用单引号引用。使用时须指定大小,如 BINARY(20) 定义了最长为 20 个单字节字符的字符串,每个字符占 1 字节的存储空间,总共固定占用 20 字节的空间,此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示,即 `\'`
- SQL 语句中的数值类型将依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型,因此在使用时要注意相应类型越界的情况。例如,9999999999999999999 会认为超过长整型的上边界而溢出,而 9999999999999999999.0 会被认为是有效的浮点数。 - SQL 语句中的数值类型将依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型,因此在使用时要注意相应类型越界的情况。例如,9999999999999999999 会认为超过长整型的上边界而溢出,而 9999999999999999999.0 会被认为是有效的浮点数。
::: :::
## 常量 ## 常量
TDengine支持多个类型的常量,细节如下表:
TDengine 支持多个类型的常量,细节如下表:
| # | **语法** | **类型** | **说明** |
| --- | :-------: | --------- | -------------------------------------- | | # | **语法** | **类型** | **说明** |
| 1 | [{+ \| -}]123 | BIGINT | 整型数值的字面量的类型均为BIGINT。如果用户输入超过了BIGINT的表示范围,TDengine 按BIGINT对数值进行截断。| | --- | :-----------------------------------------------: | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
| 2 | 123.45 | DOUBLE | 浮点数值的字面量的类型均为DOUBLE。TDengine依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型。| | 1 | [{+ \| -}]123 | BIGINT | 整型数值的字面量的类型均为 BIGINT。如果用户输入超过了 BIGINT 的表示范围,TDengine 按 BIGINT 对数值进行截断。 |
| 3 | 1.2E3 | DOUBLE | 科学计数法的字面量的类型为DOUBLE。| | 2 | 123.45 | DOUBLE | 浮点数值的字面量的类型均为 DOUBLE。TDengine 依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型。 |
| 4 | 'abc' | BINARY | 单引号括住的内容为字符串字面值,其类型为BINARY,BINARY的size为实际的字符个数。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示,即 \'。| | 3 | 1.2E3 | DOUBLE | 科学计数法的字面量的类型为 DOUBLE。 |
| 5 | "abc" | BINARY | 双引号括住的内容为字符串字面值,其类型为BINARY,BINARY的size为实际的字符个数。对于字符串内的双引号,可以用转义字符反斜线加单引号来表示,即 \"。 | | 4 | 'abc' | BINARY | 单引号括住的内容为字符串字面值,其类型为 BINARY,BINARY 的 Size 为实际的字符个数。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示,即 `\'`。 |
| 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | TIMESTAMP关键字表示后面的字符串字面量需要被解释为TIMESTAMP类型。字符串需要满足YYYY-MM-DD HH:mm:ss.MS格式,其时间分辨率为当前数据库的时间分辨率。 | | 5 | "abc" | BINARY | 双引号括住的内容为字符串字面值,其类型为 BINARY,BINARY 的 Size 为实际的字符个数。对于字符串内的双引号,可以用转义字符反斜线加单引号来表示,即 `\"`。 |
| 7 | {TRUE \| FALSE} | BOOL | 布尔类型字面量。 | | 6 | TIMESTAMP {'literal' \| "literal"} | TIMESTAMP | TIMESTAMP 关键字表示后面的字符串字面量需要被解释为 TIMESTAMP 类型。字符串需要满足 YYYY-MM-DD HH:mm:ss.MS 格式,其时间分辨率为当前数据库的时间分辨率。 |
| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | 空值字面量。可以用于任意类型。| | 7 | {TRUE \| FALSE} | BOOL | 布尔类型字面量。 |
| 8 | {'' \| "" \| '\t' \| "\t" \| ' ' \| " " \| NULL } | -- | 空值字面量。可以用于任意类型。 |
:::note :::note
- TDengine依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型,因此在使用时要注意相应类型越界的情况。例如,9999999999999999999会认为超过长整型的上边界而溢出,而9999999999999999999.0会被认为是有效的浮点数。
- TDengine 依据是否存在小数点,或使用科学计数法表示,来判断数值类型是否为整型或者浮点型,因此在使用时要注意相应类型越界的情况。例如,9999999999999999999 会认为超过长整型的上边界而溢出,而 9999999999999999999.0 会被认为是有效的浮点数。
::: :::
...@@ -69,7 +69,7 @@ order_expr: ...@@ -69,7 +69,7 @@ order_expr:
### 通配符 ### 通配符
通配符 \* 可以用于代指全部列。对于普通表,结果中只有普通列。对于超级表和子表,还包含了 TAG 列。 通配符 \* 可以用于代指全部列。对于普通表和子表,结果中只有普通列。对于超级表,还包含了 TAG 列。
```sql ```sql
SELECT * FROM d1001; SELECT * FROM d1001;
......
...@@ -1233,7 +1233,7 @@ SELECT SERVER_VERSION(); ...@@ -1233,7 +1233,7 @@ SELECT SERVER_VERSION();
### SERVER_STATUS ### SERVER_STATUS
```sql ```sql
SELECT SERVER_VERSION(); SELECT SERVER_STATUS();
``` ```
**说明**:返回服务端当前的状态。 **说明**:返回服务端当前的状态。
...@@ -6,6 +6,9 @@ description: TDengine 发布历史、Release Notes 及下载链接 ...@@ -6,6 +6,9 @@ description: TDengine 发布历史、Release Notes 及下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.1.2
<Release type="tdengine" version="3.0.1.2" />
## 3.0.1.1 ## 3.0.1.1
......
...@@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 ...@@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.2.2
<Release type="tools" version="2.2.2" />
## 2.2.0 ## 2.2.0
<Release type="tools" version="2.2.0" /> <Release type="tools" version="2.2.0" />
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
#include <stdarg.h> #include <stdarg.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include "../../../../include/client/taos.h" #include "taos.h"
#include "lauxlib.h" #include "lauxlib.h"
#include "lua.h" #include "lua.h"
#include "lualib.h" #include "lualib.h"
...@@ -35,7 +35,7 @@ static int l_connect(lua_State *L){ ...@@ -35,7 +35,7 @@ static int l_connect(lua_State *L){
} }
lua_getfield(L, 1, "port"); lua_getfield(L, 1, "port");
if (lua_isnumber(L,-1)){ if (lua_isnumber(L, -1)){
port = lua_tonumber(L, -1); port = lua_tonumber(L, -1);
//printf("port = %d\n", port); //printf("port = %d\n", port);
} }
...@@ -60,7 +60,6 @@ static int l_connect(lua_State *L){ ...@@ -60,7 +60,6 @@ static int l_connect(lua_State *L){
lua_settop(L,0); lua_settop(L,0);
taos_init();
lua_newtable(L); lua_newtable(L);
int table_index = lua_gettop(L); int table_index = lua_gettop(L);
...@@ -102,7 +101,7 @@ static int l_query(lua_State *L){ ...@@ -102,7 +101,7 @@ static int l_query(lua_State *L){
printf("failed, reason:%s\n", taos_errstr(result)); printf("failed, reason:%s\n", taos_errstr(result));
lua_pushinteger(L, -1); lua_pushinteger(L, -1);
lua_setfield(L, table_index, "code"); lua_setfield(L, table_index, "code");
lua_pushstring(L, taos_errstr(taos)); lua_pushstring(L, taos_errstr(result));
lua_setfield(L, table_index, "error"); lua_setfield(L, table_index, "error");
return 1; return 1;
...@@ -113,7 +112,6 @@ static int l_query(lua_State *L){ ...@@ -113,7 +112,6 @@ static int l_query(lua_State *L){
int rows = 0; int rows = 0;
int num_fields = taos_field_count(result); int num_fields = taos_field_count(result);
const TAOS_FIELD *fields = taos_fetch_fields(result); const TAOS_FIELD *fields = taos_fetch_fields(result);
//char temp[256];
const int affectRows = taos_affected_rows(result); const int affectRows = taos_affected_rows(result);
// printf(" affect rows:%d\r\n", affectRows); // printf(" affect rows:%d\r\n", affectRows);
...@@ -122,7 +120,7 @@ static int l_query(lua_State *L){ ...@@ -122,7 +120,7 @@ static int l_query(lua_State *L){
lua_pushinteger(L, affectRows); lua_pushinteger(L, affectRows);
lua_setfield(L, table_index, "affected"); lua_setfield(L, table_index, "affected");
lua_newtable(L); lua_newtable(L);
while ((row = taos_fetch_row(result))) { while ((row = taos_fetch_row(result))) {
//printf("row index:%d\n",rows); //printf("row index:%d\n",rows);
rows++; rows++;
...@@ -136,7 +134,7 @@ static int l_query(lua_State *L){ ...@@ -136,7 +134,7 @@ static int l_query(lua_State *L){
} }
lua_pushstring(L,fields[i].name); lua_pushstring(L,fields[i].name);
int32_t* length = taos_fetch_lengths(result);
switch (fields[i].type) { switch (fields[i].type) {
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
lua_pushinteger(L,*((char *)row[i])); lua_pushinteger(L,*((char *)row[i]));
...@@ -158,7 +156,8 @@ static int l_query(lua_State *L){ ...@@ -158,7 +156,8 @@ static int l_query(lua_State *L){
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
lua_pushstring(L,(char *)row[i]); //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]);
lua_pushlstring(L,(char *)row[i], length[i]);
break; break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
lua_pushinteger(L,*((int64_t *)row[i])); lua_pushinteger(L,*((int64_t *)row[i]));
...@@ -166,6 +165,7 @@ static int l_query(lua_State *L){ ...@@ -166,6 +165,7 @@ static int l_query(lua_State *L){
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
lua_pushinteger(L,*((char *)row[i])); lua_pushinteger(L,*((char *)row[i]));
break; break;
case TSDB_DATA_TYPE_NULL:
default: default:
lua_pushnil(L); lua_pushnil(L);
break; break;
...@@ -235,112 +235,6 @@ static int l_async_query(lua_State *L){ ...@@ -235,112 +235,6 @@ static int l_async_query(lua_State *L){
return 1; return 1;
} }
void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){
struct cb_param* p = (struct cb_param*) param;
TAOS_FIELD *fields = taos_fetch_fields(result);
int numFields = taos_num_fields(result);
// printf("\nnumfields:%d\n", numFields);
//printf("\n\r-----------------------------------------------------------------------------------\n");
lua_State *L = p->state;
lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback);
lua_newtable(L);
for (int i = 0; i < numFields; ++i) {
if (row[i] == NULL) {
continue;
}
lua_pushstring(L,fields[i].name);
switch (fields[i].type) {
case TSDB_DATA_TYPE_TINYINT:
lua_pushinteger(L,*((char *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
lua_pushinteger(L,*((short *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
lua_pushinteger(L,*((int *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
lua_pushinteger(L,*((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT:
lua_pushnumber(L,*((float *)row[i]));
break;
case TSDB_DATA_TYPE_DOUBLE:
lua_pushnumber(L,*((double *)row[i]));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
lua_pushstring(L,(char *)row[i]);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
lua_pushinteger(L,*((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_BOOL:
lua_pushinteger(L,*((char *)row[i]));
break;
default:
lua_pushnil(L);
break;
}
lua_settable(L, -3);
}
lua_call(L, 1, 0);
// printf("-----------------------------------------------------------------------------------\n\r");
}
static int l_open_stream(lua_State *L){
int r = luaL_ref(L, LUA_REGISTRYINDEX);
TAOS * taos = (TAOS*)lua_topointer(L,1);
const char * sqlstr = lua_tostring(L,2);
int stime = luaL_checknumber(L,3);
lua_newtable(L);
int table_index = lua_gettop(L);
struct cb_param *p = malloc(sizeof(struct cb_param));
p->state = L;
p->callback=r;
// printf("r:%d, L:%d\n",r,L);
void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL);
if (s == NULL) {
printf("failed to open stream, reason:%s\n", taos_errstr(taos));
free(p);
lua_pushnumber(L, -1);
lua_setfield(L, table_index, "code");
lua_pushstring(L, taos_errstr(taos));
lua_setfield(L, table_index, "error");
lua_pushlightuserdata(L,NULL);
lua_setfield(L, table_index, "stream");
}else{
// printf("success to open stream\n");
lua_pushnumber(L, 0);
lua_setfield(L, table_index, "code");
lua_pushstring(L, taos_errstr(taos));
lua_setfield(L, table_index, "error");
p->stream = s;
lua_pushlightuserdata(L,p);
lua_setfield(L, table_index, "stream");//stream has different content in lua and c.
}
return 1;
}
static int l_close_stream(lua_State *L){
//TODO:get stream and free cb_param
struct cb_param *p = lua_touserdata(L,1);
taos_close_stream(p->stream);
free(p);
return 0;
}
static int l_close(lua_State *L){ static int l_close(lua_State *L){
TAOS *taos= (TAOS*)lua_topointer(L,1); TAOS *taos= (TAOS*)lua_topointer(L,1);
...@@ -367,8 +261,6 @@ static const struct luaL_Reg lib[] = { ...@@ -367,8 +261,6 @@ static const struct luaL_Reg lib[] = {
{"query", l_query}, {"query", l_query},
{"query_a",l_async_query}, {"query_a",l_async_query},
{"close", l_close}, {"close", l_close},
{"open_stream", l_open_stream},
{"close_stream", l_close_stream},
{NULL, NULL} {NULL, NULL}
}; };
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
#include <lua.h> #include <lua.h>
#include <lauxlib.h> #include <lauxlib.h>
#include <lualib.h> #include <lualib.h>
#include "taos.h" #include <taos.h>
struct cb_param{ struct cb_param{
lua_State* state; lua_State* state;
...@@ -60,6 +60,8 @@ static int l_connect(lua_State *L){ ...@@ -60,6 +60,8 @@ static int l_connect(lua_State *L){
lua_settop(L,0); lua_settop(L,0);
taos_init();
lua_newtable(L); lua_newtable(L);
int table_index = lua_gettop(L); int table_index = lua_gettop(L);
......
...@@ -9,6 +9,50 @@ local config = { ...@@ -9,6 +9,50 @@ local config = {
max_packet_size = 1024 * 1024 max_packet_size = 1024 * 1024
} }
function dump(obj)
local getIndent, quoteStr, wrapKey, wrapVal, dumpObj
getIndent = function(level)
return string.rep("\t", level)
end
quoteStr = function(str)
return '"' .. string.gsub(str, '"', '\\"') .. '"'
end
wrapKey = function(val)
if type(val) == "number" then
return "[" .. val .. "]"
elseif type(val) == "string" then
return "[" .. quoteStr(val) .. "]"
else
return "[" .. tostring(val) .. "]"
end
end
wrapVal = function(val, level)
if type(val) == "table" then
return dumpObj(val, level)
elseif type(val) == "number" then
return val
elseif type(val) == "string" then
return quoteStr(val)
else
return tostring(val)
end
end
dumpObj = function(obj, level)
if type(obj) ~= "table" then
return wrapVal(obj)
end
level = level + 1
local tokens = {}
tokens[#tokens + 1] = "{"
for k, v in pairs(obj) do
tokens[#tokens + 1] = getIndent(level) .. wrapKey(k) .. " = " .. wrapVal(v, level) .. ","
end
tokens[#tokens + 1] = getIndent(level - 1) .. "}"
return table.concat(tokens, "\n")
end
return dumpObj(obj, 0)
end
local conn local conn
local res = driver.connect(config) local res = driver.connect(config)
if res.code ~=0 then if res.code ~=0 then
...@@ -37,7 +81,7 @@ else ...@@ -37,7 +81,7 @@ else
print("select db--- pass.") print("select db--- pass.")
end end
res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") res = driver.query(conn,"create table m1 (ts timestamp, speed int, owner binary(20), mark nchar(30))")
if res.code ~=0 then if res.code ~=0 then
print("create table---failed: "..res.error) print("create table---failed: "..res.error)
return return
...@@ -45,7 +89,7 @@ else ...@@ -45,7 +89,7 @@ else
print("create table--- pass.") print("create table--- pass.")
end end
res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001', 0, 'robotspace', '世界人民大团结万岁'), ('2019-09-01 00:00:00.002', 1, 'Hilink', '⾾⾿⿀⿁⿂⿃⿄⿅⿆⿇⿈⿉⿊⿋⿌⿍⿎⿏⿐⿑⿒⿓⿔⿕'),('2019-09-01 00:00:00.003', 2, 'Harmony', '₠₡₢₣₤₥₦₧₨₩₪₫€₭₮₯₰₱₲₳₴₵')")
if res.code ~=0 then if res.code ~=0 then
print("insert records failed: "..res.error) print("insert records failed: "..res.error)
return return
...@@ -64,21 +108,25 @@ if res.code ~=0 then ...@@ -64,21 +108,25 @@ if res.code ~=0 then
return return
else else
if (#(res.item) == 3) then if (#(res.item) == 3) then
print("select--- pass") print("select--- pass")
print(res.item[1].mark)
print(res.item[2].mark)
print(res.item[3].mark)
else else
print("select--- failed: expect 3 affected records, actually received "..#(res.item)) print("select--- failed: expect 3 affected records, actually received "..#(res.item))
end end
end end
res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") res = driver.query(conn,"create table thermometer (ts timestamp, degree double) tags(location binary(20), type int)")
if res.code ~=0 then if res.code ~=0 then
print(res.error) print(res.error)
return return
else else
print("create super table--- pass") print("create super table--- pass")
end end
res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") res = driver.query(conn,"create table therm1 using thermometer tags ('beijing', 1)")
if res.code ~=0 then if res.code ~=0 then
print(res.error) print(res.error)
return return
...@@ -86,7 +134,7 @@ else ...@@ -86,7 +134,7 @@ else
print("create table--- pass") print("create table--- pass")
end end
res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") res = driver.query(conn,"insert into therm1 values ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)")
if res.code ~=0 then if res.code ~=0 then
print(res.error) print(res.error)
...@@ -99,14 +147,14 @@ else ...@@ -99,14 +147,14 @@ else
end end
end end
res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") res = driver.query(conn,"select count(*) as cnt, avg(degree) as av, max(degree), min(degree) from thermometer where location='beijing' or location='tianjin' group by location, type")
if res.code ~=0 then if res.code ~=0 then
print("select from super table--- failed:"..res.error) print("select from super table--- failed:"..res.error)
return return
else else
print("select from super table--- pass") print("select from super table--- pass")
for i = 1, #(res.item) do for i = 1, #(res.item) do
print("res:"..res.item[i].count) print("res:"..res.item[i].cnt)
end end
end end
...@@ -127,30 +175,13 @@ end ...@@ -127,30 +175,13 @@ end
driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback)
res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)")
function stream_callback(t) print("From now on we start continous insert in an definite loop, pls wait for about 10 seconds and check stream table for result.")
print("------------------------")
print("continuous query result:")
for key, value in pairs(t) do
print("key:"..key..", value:"..value)
end
end
local stream
res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0, stream_callback)
if res.code ~=0 then
print("open stream--- failed:"..res.error)
return
else
print("open stream--- pass")
stream = res.stream
end
print("From now on we start continous insert in an definite (infinite if you want) loop.")
local loop_index = 0 local loop_index = 0
while loop_index < 30 do while loop_index < 10 do
local t = os.time()*1000 local t = os.time()*1000
local v = loop_index local v = math.random(20)
res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v))
if res.code ~=0 then if res.code ~=0 then
...@@ -162,7 +193,5 @@ while loop_index < 30 do ...@@ -162,7 +193,5 @@ while loop_index < 30 do
os.execute("sleep " .. 1) os.execute("sleep " .. 1)
loop_index = loop_index + 1 loop_index = loop_index + 1
end end
driver.query(conn,"DROP STREAM IF EXISTS avg_therm_s")
driver.close_stream(stream)
driver.close(conn) driver.close(conn)
...@@ -116,6 +116,7 @@ enum { ...@@ -116,6 +116,7 @@ enum {
STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__GET_RES, STREAM_INPUT__GET_RES,
STREAM_INPUT__CHECKPOINT, STREAM_INPUT__CHECKPOINT,
STREAM_INPUT__REF_DATA_BLOCK,
STREAM_INPUT__DESTROY, STREAM_INPUT__DESTROY,
}; };
......
...@@ -96,6 +96,8 @@ extern int32_t tsQueryPolicy; ...@@ -96,6 +96,8 @@ extern int32_t tsQueryPolicy;
extern int32_t tsQuerySmaOptimize; extern int32_t tsQuerySmaOptimize;
extern int32_t tsQueryRsmaTolerance; extern int32_t tsQueryRsmaTolerance;
extern bool tsQueryPlannerTrace; extern bool tsQueryPlannerTrace;
extern int32_t tsQueryNodeChunkSize;
extern bool tsQueryUseNodeAllocator;
// client // client
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;
......
...@@ -275,6 +275,17 @@ typedef struct SNodeList { ...@@ -275,6 +275,17 @@ typedef struct SNodeList {
SListCell* pTail; SListCell* pTail;
} SNodeList; } SNodeList;
typedef struct SNodeAllocator SNodeAllocator;
int32_t nodesInitAllocatorSet();
void nodesDestroyAllocatorSet();
int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId);
int32_t nodesAcquireAllocator(int64_t allocatorId);
int32_t nodesReleaseAllocator(int64_t allocatorId);
int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId);
int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId);
void nodesDestroyAllocator(int64_t allocatorId);
SNode* nodesMakeNode(ENodeType type); SNode* nodesMakeNode(ENodeType type);
void nodesDestroyNode(SNode* pNode); void nodesDestroyNode(SNode* pNode);
......
...@@ -56,6 +56,7 @@ typedef struct SParseContext { ...@@ -56,6 +56,7 @@ typedef struct SParseContext {
bool nodeOffline; bool nodeOffline;
SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
} SParseContext; } SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
......
...@@ -39,6 +39,7 @@ typedef struct SPlanContext { ...@@ -39,6 +39,7 @@ typedef struct SPlanContext {
int32_t msgLen; int32_t msgLen;
const char* pUser; const char* pUser;
bool sysInfo; bool sysInfo;
int64_t allocatorId;
} SPlanContext; } SPlanContext;
// Create the physical plan for the query, according to the AST. // Create the physical plan for the query, according to the AST.
......
...@@ -67,6 +67,7 @@ typedef struct SSchedulerReq { ...@@ -67,6 +67,7 @@ typedef struct SSchedulerReq {
SRequestConnInfo *pConn; SRequestConnInfo *pConn;
SArray *pNodeList; SArray *pNodeList;
SQueryPlan *pDag; SQueryPlan *pDag;
int64_t allocatorRefId;
const char *sql; const char *sql;
int64_t startTs; int64_t startTs;
schedulerExecFp execFp; schedulerExecFp execFp;
......
...@@ -125,6 +125,14 @@ typedef struct { ...@@ -125,6 +125,14 @@ typedef struct {
SArray* blocks; // SArray<SSDataBlock> SArray* blocks; // SArray<SSDataBlock>
} SStreamDataBlock; } SStreamDataBlock;
// ref data block, for delete
typedef struct {
int8_t type;
int64_t ver;
int32_t* dataRef;
SSDataBlock* pBlock;
} SStreamRefDataBlock;
typedef struct { typedef struct {
int8_t type; int8_t type;
} SStreamCheckpoint; } SStreamCheckpoint;
...@@ -339,7 +347,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem ...@@ -339,7 +347,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data);
taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
// qStreamInput(pTask->exec.executor, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone);
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE ||
pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
taosWriteQitem(pTask->inputQueue->queue, pItem); taosWriteQitem(pTask->inputQueue->queue, pItem);
// qStreamInput(pTask->exec.executor, pItem); // qStreamInput(pTask->exec.executor, pItem);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT) { } else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
...@@ -492,7 +501,9 @@ typedef struct { ...@@ -492,7 +501,9 @@ typedef struct {
int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);
int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
void tFreeStreamDispatchReq(SStreamDispatchReq* pReq); void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq);
void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq);
int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamSetupTrigger(SStreamTask* pTask);
......
...@@ -22,6 +22,7 @@ extern "C" { ...@@ -22,6 +22,7 @@ extern "C" {
#include "cJSON.h" #include "cJSON.h"
#include "tdef.h" #include "tdef.h"
#include "tlrucache.h"
#include "tmsgcb.h" #include "tmsgcb.h"
extern bool gRaftDetailLog; extern bool gRaftDetailLog;
...@@ -153,7 +154,8 @@ typedef struct SSyncFSM { ...@@ -153,7 +154,8 @@ typedef struct SSyncFSM {
// abstract definition of log store in raft // abstract definition of log store in raft
// SWal implements it // SWal implements it
typedef struct SSyncLogStore { typedef struct SSyncLogStore {
void* data; SLRUCache* pCache;
void* data;
// append one log entry // append one log entry
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
......
...@@ -202,8 +202,8 @@ elif [[ ${packgeName} =~ "tar" ]];then ...@@ -202,8 +202,8 @@ elif [[ ${packgeName} =~ "tar" ]];then
cd ${oriInstallPath}/${originTdpPath} && tar xf ${subFile} cd ${oriInstallPath}/${originTdpPath} && tar xf ${subFile}
fi fi
cd ${oriInstallPath}/${originTdpPath} && tree > ${installPath}/base_${originversion}_checkfile cd ${oriInstallPath}/${originTdpPath} && tree -I "driver" > ${installPath}/base_${originversion}_checkfile
cd ${installPath}/${tdPath} && tree > ${installPath}/now_${version}_checkfile cd ${installPath}/${tdPath} && tree -I "driver" > ${installPath}/now_${version}_checkfile
cd ${installPath} cd ${installPath}
diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log
...@@ -215,6 +215,7 @@ elif [[ ${packgeName} =~ "tar" ]];then ...@@ -215,6 +215,7 @@ elif [[ ${packgeName} =~ "tar" ]];then
exit -1 exit -1
else else
echoColor G "The number and names of files are the same as previous installation packages" echoColor G "The number and names of files are the same as previous installation packages"
rm -rf ${installPath}/diffFile.log
fi fi
echoColor YD "===== install Package of tar =====" echoColor YD "===== install Package of tar ====="
cd ${installPath}/${tdPath} cd ${installPath}/${tdPath}
...@@ -251,6 +252,9 @@ if [[ ${packgeName} =~ "server" ]] ;then ...@@ -251,6 +252,9 @@ if [[ ${packgeName} =~ "server" ]] ;then
systemctl restart taosd systemctl restart taosd
fi fi
rm -rf ${installPath}/${packgeName}
rm -rf ${installPath}/${tdPath}/
# if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then # if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then
# echoColor G "===== install taos-tools when package is lite or client =====" # echoColor G "===== install taos-tools when package is lite or client ====="
# cd ${installPath} # cd ${installPath}
......
...@@ -250,6 +250,7 @@ typedef struct SRequestObj { ...@@ -250,6 +250,7 @@ typedef struct SRequestObj {
bool inRetry; bool inRetry;
uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog
uint32_t retry; uint32_t retry;
int64_t allocatorRefId;
} SRequestObj; } SRequestObj;
typedef struct SSyncQueryParam { typedef struct SSyncQueryParam {
......
...@@ -288,6 +288,7 @@ void *createRequest(uint64_t connId, int32_t type) { ...@@ -288,6 +288,7 @@ void *createRequest(uint64_t connId, int32_t type) {
pRequest->body.resInfo.convertUcs4 = true; // convert ucs4 by default pRequest->body.resInfo.convertUcs4 = true; // convert ucs4 by default
pRequest->type = type; pRequest->type = type;
pRequest->allocatorRefId = -1;
pRequest->pDb = getDbOfConnection(pTscObj); pRequest->pDb = getDbOfConnection(pTscObj);
pRequest->pTscObj = pTscObj; pRequest->pTscObj = pTscObj;
...@@ -349,6 +350,7 @@ void doDestroyRequest(void *p) { ...@@ -349,6 +350,7 @@ void doDestroyRequest(void *p) {
taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->tableList);
taosArrayDestroy(pRequest->dbList); taosArrayDestroy(pRequest->dbList);
taosArrayDestroy(pRequest->targetTableList); taosArrayDestroy(pRequest->targetTableList);
nodesDestroyAllocator(pRequest->allocatorRefId);
destroyQueryExecRes(&pRequest->body.resInfo.execRes); destroyQueryExecRes(&pRequest->body.resInfo.execRes);
...@@ -411,6 +413,7 @@ void taos_init_imp(void) { ...@@ -411,6 +413,7 @@ void taos_init_imp(void) {
initTaskQueue(); initTaskQueue();
fmFuncMgtInit(); fmFuncMgtInit();
nodesInitAllocatorSet();
clientConnRefPool = taosOpenRef(200, destroyTscObj); clientConnRefPool = taosOpenRef(200, destroyTscObj);
clientReqRefPool = taosOpenRef(40960, doDestroyRequest); clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
......
...@@ -195,6 +195,19 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, ...@@ -195,6 +195,19 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
(*pRequest)->allocatorRefId = -1;
if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) {
if (TSDB_CODE_SUCCESS !=
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self,
(*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest);
*pRequest = NULL;
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
}
tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1023,7 +1036,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM ...@@ -1023,7 +1036,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
.pMsg = pRequest->msgBuf, .pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user, .pUser = pRequest->pTscObj->user,
.sysInfo = pRequest->pTscObj->sysInfo}; .sysInfo = pRequest->pTscObj->sysInfo,
.allocatorId = pRequest->allocatorRefId};
SAppInstInfo* pAppInfo = getAppInfo(pRequest); SAppInstInfo* pAppInfo = getAppInfo(pRequest);
SQueryPlan* pDag = NULL; SQueryPlan* pDag = NULL;
...@@ -1048,6 +1062,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM ...@@ -1048,6 +1062,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
.pConn = &conn, .pConn = &conn,
.pNodeList = pNodeList, .pNodeList = pNodeList,
.pDag = pDag, .pDag = pDag,
.allocatorRefId = pRequest->allocatorRefId,
.sql = pRequest->sqlstr, .sql = pRequest->sqlstr,
.startTs = pRequest->metric.start, .startTs = pRequest->metric.start,
.execFp = schedulerExecCb, .execFp = schedulerExecCb,
......
...@@ -65,6 +65,7 @@ void taos_cleanup(void) { ...@@ -65,6 +65,7 @@ void taos_cleanup(void) {
fmFuncMgtDestroy(); fmFuncMgtDestroy();
qCleanupKeywordsTable(); qCleanupKeywordsTable();
nodesDestroyAllocatorSet();
id = clientConnRefPool; id = clientConnRefPool;
clientConnRefPool = -1; clientConnRefPool = -1;
...@@ -775,7 +776,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { ...@@ -775,7 +776,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
.enableSysInfo = pTscObj->sysInfo, .enableSysInfo = pTscObj->sysInfo,
.async = true, .async = true,
.svrVer = pTscObj->sVer, .svrVer = pTscObj->sVer,
.nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes)}; .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
.allocatorId = pRequest->allocatorRefId};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
...@@ -93,6 +93,8 @@ int32_t tsQueryPolicy = 1; ...@@ -93,6 +93,8 @@ int32_t tsQueryPolicy = 1;
int32_t tsQuerySmaOptimize = 0; int32_t tsQuerySmaOptimize = 0;
int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data.
bool tsQueryPlannerTrace = false; bool tsQueryPlannerTrace = false;
int32_t tsQueryNodeChunkSize = 32 * 1024;
bool tsQueryUseNodeAllocator = true;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
...@@ -286,6 +288,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { ...@@ -286,6 +288,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1;
if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1;
if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1;
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
...@@ -647,6 +651,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { ...@@ -647,6 +651,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval;
return 0; return 0;
} }
...@@ -982,6 +988,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { ...@@ -982,6 +988,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
} else if (strcasecmp("queryPlannerTrace", name) == 0) { } else if (strcasecmp("queryPlannerTrace", name) == 0) {
tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
} else if (strcasecmp("queryNodeChunkSize", name) == 0) {
tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
} else if (strcasecmp("queryUseNodeAllocator", name) == 0) {
tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval;
} else if (strcasecmp("queryRsmaTolerance", name) == 0) { } else if (strcasecmp("queryRsmaTolerance", name) == 0) {
tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32;
} }
......
...@@ -362,8 +362,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ...@@ -362,8 +362,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
// update uid index // update uid index
metaUpdateUidIdx(pMeta, &nStbEntry); metaUpdateUidIdx(pMeta, &nStbEntry);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
metaULock(pMeta); metaULock(pMeta);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbTbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbTbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
...@@ -922,6 +923,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ...@@ -922,6 +923,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
} }
metaWLock(pMeta);
// save to table.db // save to table.db
metaSaveToTbDb(pMeta, &ctbEntry); metaSaveToTbDb(pMeta, &ctbEntry);
...@@ -936,6 +939,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ...@@ -936,6 +939,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
metaULock(pMeta);
tDecoderClear(&dc1); tDecoderClear(&dc1);
tDecoderClear(&dc2); tDecoderClear(&dc2);
if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
......
...@@ -59,7 +59,7 @@ static void destroySTqHandle(void* data) { ...@@ -59,7 +59,7 @@ static void destroySTqHandle(void* data) {
tqCloseReader(pData->execHandle.pExecReader); tqCloseReader(pData->execHandle.pExecReader);
walCloseReader(pData->pWalReader); walCloseReader(pData->pWalReader);
taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid);
} else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE){ } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
walCloseReader(pData->pWalReader); walCloseReader(pData->pWalReader);
tqCloseReader(pData->execHandle.pExecReader); tqCloseReader(pData->execHandle.pExecReader);
} }
...@@ -664,7 +664,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe ...@@ -664,7 +664,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe
tqError("vgId:%d, build new consumer handle %s for consumer %d, but old consumerId is %ld", req.vgId, req.subKey, tqError("vgId:%d, build new consumer handle %s for consumer %d, but old consumerId is %ld", req.vgId, req.subKey,
req.newConsumerId, req.oldConsumerId); req.newConsumerId, req.oldConsumerId);
} }
ASSERT(req.newConsumerId != -1); if (req.newConsumerId == -1) {
tqError("vgId:%d, tq invalid rebalance request, new consumerId %ld", req.vgId, req.newConsumerId);
return 0;
}
STqHandle tqHandle = {0}; STqHandle tqHandle = {0};
pHandle = &tqHandle; pHandle = &tqHandle;
/*taosInitRWLatch(&pExec->lock);*/ /*taosInitRWLatch(&pExec->lock);*/
...@@ -876,6 +879,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { ...@@ -876,6 +879,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
taosArrayDestroy(pRes->uidList); taosArrayDestroy(pRes->uidList);
int32_t* pRef = taosMemoryMalloc(sizeof(int32_t));
*pRef = 1;
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
...@@ -885,6 +891,33 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { ...@@ -885,6 +891,33 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver);
if (!failed) {
SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM);
pRefBlock->type = STREAM_INPUT__REF_DATA_BLOCK;
pRefBlock->pBlock = pDelBlock;
pRefBlock->dataRef = pRef;
atomic_add_fetch_32(pRefBlock->dataRef, 1);
if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
qError("stream task input del failed, task id %d", pTask->taskId);
continue;
}
if (streamSchedExec(pTask) < 0) {
qError("stream task launch failed, task id %d", pTask->taskId);
continue;
}
} else {
streamTaskInputFail(pTask);
}
}
int32_t ref = atomic_sub_fetch_32(pRef, 1);
ASSERT(ref >= 0);
if (ref == 0) {
taosMemoryFree(pDelBlock);
taosMemoryFree(pRef);
}
#if 0
SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; pStreamBlock->type = STREAM_INPUT__DATA_BLOCK;
pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock));
...@@ -908,6 +941,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { ...@@ -908,6 +941,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
} }
} }
blockDataDestroy(pDelBlock); blockDataDestroy(pDelBlock);
#endif
return 0; return 0;
} }
...@@ -1045,6 +1079,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { ...@@ -1045,6 +1079,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, msgBody, msgLen); tDecoderInit(&decoder, msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req); tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder);
int32_t taskId = req.dstTaskId; int32_t taskId = req.dstTaskId;
SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
if (pTask) { if (pTask) {
...@@ -1053,6 +1088,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { ...@@ -1053,6 +1088,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
.code = 0, .code = 0,
}; };
streamProcessRetrieveReq(pTask, &req, &rsp); streamProcessRetrieveReq(pTask, &req, &rsp);
tDeleteStreamRetrieveReq(&req);
return 0; return 0;
} else { } else {
return -1; return -1;
......
...@@ -676,6 +676,18 @@ static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsm ...@@ -676,6 +676,18 @@ static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsm
static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { static void vnodeRestoreFinish(struct SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
do {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
if (itemSize == 0) {
vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId);
break;
} else {
vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId);
taosMsleep(10);
}
} while (true);
pVnode->restored = true; pVnode->restored = true;
vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); vDebug("vgId:%d, sync restore finished", pVnode->config.vgId);
} }
......
...@@ -1480,6 +1480,40 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { ...@@ -1480,6 +1480,40 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
} }
} }
static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, SStreamScanInfo* pInfo) {
STqReader* pReader = pInfo->tqReader;
int32_t rows = pSrc->info.rows;
blockDataEnsureCapacity(pDst, rows);
SColumnInfoData* pSrcStartCol = taosArrayGet(pSrc->pDataBlock, START_TS_COLUMN_INDEX);
uint64_t* startCol = (uint64_t*)pSrcStartCol->pData;
SColumnInfoData* pSrcEndCol = taosArrayGet(pSrc->pDataBlock, END_TS_COLUMN_INDEX);
uint64_t* endCol = (uint64_t*)pSrcEndCol->pData;
SColumnInfoData* pSrcUidCol = taosArrayGet(pSrc->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pSrcUidCol->pData;
SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX);
int32_t j = 0;
for (int32_t i = 0; i < rows; i++) {
if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) {
colDataAppend(pDstStartCol, j, (const char*)&startCol[i], false);
colDataAppend(pDstEndCol, j, (const char*)&endCol[i], false);
colDataAppend(pDstUidCol, j, (const char*)&uidCol[i], false);
colDataAppendNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j);
colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j);
colDataAppendNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j);
j++;
}
}
pDst->info = pSrc->info;
pDst->info.rows = j;
return 0;
}
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
...@@ -1568,6 +1602,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { ...@@ -1568,6 +1602,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
} break; } break;
case STREAM_DELETE_DATA: { case STREAM_DELETE_DATA: {
printDataBlock(pBlock, "stream scan delete recv"); printDataBlock(pBlock, "stream scan delete recv");
if (pInfo->tqReader) {
SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA);
filterDelBlockByUid(pDelBlock, pBlock, pInfo);
pBlock = pDelBlock;
}
printDataBlock(pBlock, "stream scan delete recv filtered");
if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes); generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes);
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;
...@@ -1647,6 +1687,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { ...@@ -1647,6 +1687,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
while (1) { while (1) {
if (pInfo->tqReader->pMsg == NULL) { if (pInfo->tqReader->pMsg == NULL) {
if (pInfo->validBlockIndex >= totBlockNum) { if (pInfo->validBlockIndex >= totBlockNum) {
updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
return NULL; return NULL;
} }
......
...@@ -1695,6 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { ...@@ -1695,6 +1695,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
} }
nodesDestroyNode((SNode*)pInfo->pPhyNode); nodesDestroyNode((SNode*)pInfo->pPhyNode);
colDataDestroy(&pInfo->twAggSup.timeWindowData); colDataDestroy(&pInfo->twAggSup.timeWindowData);
cleanupGroupResInfo(&pInfo->groupResInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
...@@ -3073,6 +3074,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { ...@@ -3073,6 +3074,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
taosArrayRemove(chArray, index); taosArrayRemove(chArray, index);
if (taosArrayGetSize(chArray) == 0) { if (taosArrayGetSize(chArray) == 0) {
// pull data is over // pull data is over
taosArrayDestroy(chArray);
taosHashRemove(pMap, &winRes, sizeof(SWinKey)); taosHashRemove(pMap, &winRes, sizeof(SWinKey));
} }
} }
...@@ -3109,9 +3111,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { ...@@ -3109,9 +3111,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
TSKEY maxTs = INT64_MIN; TSKEY maxTs = INT64_MIN;
TSKEY minTs = INT64_MAX; TSKEY minTs = INT64_MAX;
...@@ -3175,6 +3174,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { ...@@ -3175,6 +3174,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} }
} }
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
while (1) { while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
...@@ -5755,8 +5757,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { ...@@ -5755,8 +5757,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
SStreamState* pState = pTaskInfo->streamInfo.pState;
while (1) { while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) { if (pBlock == NULL) {
...@@ -5805,36 +5805,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { ...@@ -5805,36 +5805,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
} }
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs);
#if 0
if (pState) {
printf(">>>>>>>> stream read backend\n");
SWinKey key = {
.ts = 1,
.groupId = 2,
};
char* val = NULL;
int32_t sz;
if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
ASSERT(0);
}
printf("stream read %s %d\n", val, sz);
streamFreeVal(val);
SStreamStateCur* pCur = streamStateGetCur(pState, &key);
ASSERT(pCur);
while (streamStateCurNext(pState, pCur) == 0) {
SWinKey key1;
const void* val1;
if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
break;
}
printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
}
streamStateFreeCur(pCur);
}
#endif
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
pOperator); pOperator);
......
...@@ -21,9 +21,209 @@ ...@@ -21,9 +21,209 @@
#include "taoserror.h" #include "taoserror.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "thash.h" #include "thash.h"
#include "tref.h"
typedef struct SNodeMemChunk {
int32_t availableSize;
int32_t usedSize;
char* pBuf;
struct SNodeMemChunk* pNext;
} SNodeMemChunk;
typedef struct SNodeAllocator {
int64_t self;
int64_t queryId;
int32_t chunkSize;
int32_t chunkNum;
SNodeMemChunk* pCurrChunk;
SNodeMemChunk* pChunks;
TdThreadMutex mutex;
} SNodeAllocator;
static threadlocal SNodeAllocator* g_pNodeAllocator;
static int32_t g_allocatorReqRefPool = -1;
static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) {
SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize);
if (NULL == pNewChunk) {
return NULL;
}
pNewChunk->pBuf = (char*)(pNewChunk + 1);
pNewChunk->availableSize = pAllocator->chunkSize;
pNewChunk->usedSize = 0;
pNewChunk->pNext = NULL;
if (NULL != pAllocator->pCurrChunk) {
pAllocator->pCurrChunk->pNext = pNewChunk;
}
pAllocator->pCurrChunk = pNewChunk;
if (NULL == pAllocator->pChunks) {
pAllocator->pChunks = pNewChunk;
}
++(pAllocator->chunkNum);
return pNewChunk;
}
static void* nodesCallocImpl(int32_t size) {
if (NULL == g_pNodeAllocator) {
return taosMemoryCalloc(1, size);
}
if (g_pNodeAllocator->pCurrChunk->usedSize + size > g_pNodeAllocator->pCurrChunk->availableSize) {
if (NULL == callocNodeChunk(g_pNodeAllocator)) {
return NULL;
}
}
void* p = g_pNodeAllocator->pCurrChunk->pBuf + g_pNodeAllocator->pCurrChunk->usedSize;
g_pNodeAllocator->pCurrChunk->usedSize += size;
return p;
}
static void* nodesCalloc(int32_t num, int32_t size) {
void* p = nodesCallocImpl(num * size + 1);
if (NULL == p) {
return NULL;
}
*(char*)p = (NULL != g_pNodeAllocator) ? 1 : 0;
return (char*)p + 1;
}
static void nodesFree(void* p) {
char* ptr = (char*)p - 1;
if (0 == *ptr) {
taosMemoryFree(ptr);
}
return;
}
static int32_t createNodeAllocator(int32_t chunkSize, SNodeAllocator** pAllocator) {
*pAllocator = taosMemoryCalloc(1, sizeof(SNodeAllocator));
if (NULL == *pAllocator) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pAllocator)->chunkSize = chunkSize;
if (NULL == callocNodeChunk(*pAllocator)) {
taosMemoryFreeClear(*pAllocator);
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexInit(&(*pAllocator)->mutex, NULL);
return TSDB_CODE_SUCCESS;
}
static void destroyNodeAllocator(void* p) {
if (NULL == p) {
return;
}
SNodeAllocator* pAllocator = p;
nodesDebug("query id %" PRIx64 " allocator id %" PRIx64 " alloc chunkNum: %d, chunkTotakSize: %d",
pAllocator->queryId, pAllocator->self, pAllocator->chunkNum, pAllocator->chunkNum * pAllocator->chunkSize);
SNodeMemChunk* pChunk = pAllocator->pChunks;
while (NULL != pChunk) {
SNodeMemChunk* pTemp = pChunk->pNext;
taosMemoryFree(pChunk);
pChunk = pTemp;
}
taosThreadMutexDestroy(&pAllocator->mutex);
taosMemoryFree(pAllocator);
}
int32_t nodesInitAllocatorSet() {
if (g_allocatorReqRefPool >= 0) {
nodesWarn("nodes already initialized");
return TSDB_CODE_SUCCESS;
}
g_allocatorReqRefPool = taosOpenRef(1024, destroyNodeAllocator);
if (g_allocatorReqRefPool < 0) {
nodesError("init nodes failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
void nodesDestroyAllocatorSet() {
if (g_allocatorReqRefPool >= 0) {
SNodeAllocator* pAllocator = taosIterateRef(g_allocatorReqRefPool, 0);
int64_t refId = 0;
while (NULL != pAllocator) {
refId = pAllocator->self;
taosRemoveRef(g_allocatorReqRefPool, refId);
pAllocator = taosIterateRef(g_allocatorReqRefPool, refId);
}
taosCloseRef(g_allocatorReqRefPool);
}
}
int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId) {
SNodeAllocator* pAllocator = NULL;
int32_t code = createNodeAllocator(chunkSize, &pAllocator);
if (TSDB_CODE_SUCCESS == code) {
pAllocator->self = taosAddRef(g_allocatorReqRefPool, pAllocator);
if (pAllocator->self <= 0) {
return terrno;
}
pAllocator->queryId = queryId;
*pAllocatorId = pAllocator->self;
}
return code;
}
int32_t nodesAcquireAllocator(int64_t allocatorId) {
if (allocatorId <= 0) {
return TSDB_CODE_SUCCESS;
}
SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId);
if (NULL == pAllocator) {
return terrno;
}
taosThreadMutexLock(&pAllocator->mutex);
g_pNodeAllocator = pAllocator;
return TSDB_CODE_SUCCESS;
}
int32_t nodesReleaseAllocator(int64_t allocatorId) {
if (allocatorId <= 0) {
return TSDB_CODE_SUCCESS;
}
if (NULL == g_pNodeAllocator) {
nodesError("allocator id %" PRIx64
" release failed: The nodesReleaseAllocator function needs to be called after the nodesAcquireAllocator "
"function is called!",
allocatorId);
return TSDB_CODE_FAILED;
}
SNodeAllocator* pAllocator = g_pNodeAllocator;
g_pNodeAllocator = NULL;
taosThreadMutexUnlock(&pAllocator->mutex);
return taosReleaseRef(g_allocatorReqRefPool, allocatorId);
}
int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) {
if (allocatorId <= 0) {
return 0;
}
SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId);
return pAllocator->self;
}
int64_t nodesReleaseAllocatorWeakRef(int64_t allocatorId) { return taosReleaseRef(g_allocatorReqRefPool, allocatorId); }
void nodesDestroyAllocator(int64_t allocatorId) {
if (allocatorId <= 0) {
return;
}
taosRemoveRef(g_allocatorReqRefPool, allocatorId);
}
static SNode* makeNode(ENodeType type, size_t size) { static SNode* makeNode(ENodeType type, int32_t size) {
SNode* p = taosMemoryCalloc(1, size); SNode* p = nodesCalloc(1, size);
if (NULL == p) { if (NULL == p) {
return NULL; return NULL;
} }
...@@ -824,6 +1024,7 @@ void nodesDestroyNode(SNode* pNode) { ...@@ -824,6 +1024,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pLogicNode->pWStartTs); nodesDestroyNode(pLogicNode->pWStartTs);
nodesDestroyNode(pLogicNode->pValues); nodesDestroyNode(pLogicNode->pValues);
nodesDestroyList(pLogicNode->pFillExprs); nodesDestroyList(pLogicNode->pFillExprs);
nodesDestroyList(pLogicNode->pNotFillExprs);
break; break;
} }
case QUERY_NODE_LOGIC_PLAN_SORT: { case QUERY_NODE_LOGIC_PLAN_SORT: {
...@@ -1021,12 +1222,12 @@ void nodesDestroyNode(SNode* pNode) { ...@@ -1021,12 +1222,12 @@ void nodesDestroyNode(SNode* pNode) {
default: default:
break; break;
} }
taosMemoryFreeClear(pNode); nodesFree(pNode);
return; return;
} }
SNodeList* nodesMakeList() { SNodeList* nodesMakeList() {
SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); SNodeList* p = nodesCalloc(1, sizeof(SNodeList));
if (NULL == p) { if (NULL == p) {
return NULL; return NULL;
} }
...@@ -1037,7 +1238,7 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { ...@@ -1037,7 +1238,7 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) {
if (NULL == pList || NULL == pNode) { if (NULL == pList || NULL == pNode) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); SListCell* p = nodesCalloc(1, sizeof(SListCell));
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
...@@ -1104,7 +1305,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { ...@@ -1104,7 +1305,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
} }
pTarget->pTail = pSrc->pTail; pTarget->pTail = pSrc->pTail;
pTarget->length += pSrc->length; pTarget->length += pSrc->length;
taosMemoryFreeClear(pSrc); nodesFree(pSrc);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1124,7 +1325,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) { ...@@ -1124,7 +1325,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) {
if (NULL == pList || NULL == pNode) { if (NULL == pList || NULL == pNode) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); SListCell* p = nodesCalloc(1, sizeof(SListCell));
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
...@@ -1152,7 +1353,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { ...@@ -1152,7 +1353,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
} }
SListCell* pNext = pCell->pNext; SListCell* pNext = pCell->pNext;
nodesDestroyNode(pCell->pNode); nodesDestroyNode(pCell->pNode);
taosMemoryFreeClear(pCell); nodesFree(pCell);
--(pList->length); --(pList->length);
return pNext; return pNext;
} }
...@@ -1172,7 +1373,7 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) { ...@@ -1172,7 +1373,7 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) {
pPos->pPrev = pSrc->pTail; pPos->pPrev = pSrc->pTail;
pTarget->length += pSrc->length; pTarget->length += pSrc->length;
taosMemoryFreeClear(pSrc); nodesFree(pSrc);
} }
SNode* nodesListGetNode(SNodeList* pList, int32_t index) { SNode* nodesListGetNode(SNodeList* pList, int32_t index) {
...@@ -1204,7 +1405,7 @@ void nodesDestroyList(SNodeList* pList) { ...@@ -1204,7 +1405,7 @@ void nodesDestroyList(SNodeList* pList) {
while (NULL != pNext) { while (NULL != pNext) {
pNext = nodesListErase(pList, pNext); pNext = nodesListErase(pList, pNext);
} }
taosMemoryFreeClear(pList); nodesFree(pList);
} }
void nodesClearList(SNodeList* pList) { void nodesClearList(SNodeList* pList) {
...@@ -1216,9 +1417,9 @@ void nodesClearList(SNodeList* pList) { ...@@ -1216,9 +1417,9 @@ void nodesClearList(SNodeList* pList) {
while (NULL != pNext) { while (NULL != pNext) {
SListCell* tmp = pNext; SListCell* tmp = pNext;
pNext = pNext->pNext; pNext = pNext->pNext;
taosMemoryFreeClear(tmp); nodesFree(tmp);
} }
taosMemoryFreeClear(pList); nodesFree(pList);
} }
void* nodesGetValueFromNode(SValueNode* pNode) { void* nodesGetValueFromNode(SValueNode* pNode) {
......
...@@ -247,7 +247,8 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { ...@@ -247,7 +247,8 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) {
pExpr->userAlias[len] = '\0'; pExpr->userAlias[len] = '\0';
} }
} }
taosMemoryFreeClear(pNode); pRawExpr->pNode = NULL;
nodesDestroyNode(pNode);
return pRealizedExpr; return pRealizedExpr;
} }
......
...@@ -177,15 +177,18 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { ...@@ -177,15 +177,18 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) {
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
SParseMetaCache metaCache = {0}; SParseMetaCache metaCache = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { if (TSDB_CODE_SUCCESS == code) {
code = parseInsertSyntax(pCxt, pQuery, &metaCache); if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
} else { code = parseInsertSyntax(pCxt, pQuery, &metaCache);
code = parseSqlSyntax(pCxt, pQuery, &metaCache); } else {
code = parseSqlSyntax(pCxt, pQuery, &metaCache);
}
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); code = buildCatalogReq(pCxt, &metaCache, pCatalogReq);
} }
nodesReleaseAllocator(pCxt->allocatorId);
destoryParseMetaCache(&metaCache, true); destoryParseMetaCache(&metaCache, true);
terrno = code; terrno = code;
return code; return code;
...@@ -194,7 +197,10 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq ...@@ -194,7 +197,10 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq
int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq,
const struct SMetaData* pMetaData, SQuery* pQuery) { const struct SMetaData* pMetaData, SQuery* pQuery) {
SParseMetaCache metaCache = {0}; SParseMetaCache metaCache = {0};
int32_t code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (NULL == pQuery->pRoot) { if (NULL == pQuery->pRoot) {
code = parseInsertSql(pCxt, &pQuery, &metaCache); code = parseInsertSql(pCxt, &pQuery, &metaCache);
...@@ -202,6 +208,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata ...@@ -202,6 +208,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
code = analyseSemantic(pCxt, pQuery, &metaCache); code = analyseSemantic(pCxt, pQuery, &metaCache);
} }
} }
nodesReleaseAllocator(pCxt->allocatorId);
destoryParseMetaCache(&metaCache, false); destoryParseMetaCache(&metaCache, false);
terrno = code; terrno = code;
return code; return code;
......
...@@ -119,12 +119,18 @@ class ParserTestBaseImpl { ...@@ -119,12 +119,18 @@ class ParserTestBaseImpl {
TEST_INTERFACE_ASYNC_API TEST_INTERFACE_ASYNC_API
}; };
static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { static void destoryParseContext(SParseContext* pCxt) {
taosArrayDestroy(pCxt->pTableMetaPos);
taosArrayDestroy(pCxt->pTableVgroupPos);
delete pCxt;
}
static void destoryParseMetaCacheWarpper(SParseMetaCache* pMetaCache, bool request) {
destoryParseMetaCache(pMetaCache, request); destoryParseMetaCache(pMetaCache, request);
delete pMetaCache; delete pMetaCache;
} }
static void _destroyQuery(SQuery** pQuery) { static void destroyQuery(SQuery** pQuery) {
if (nullptr == pQuery) { if (nullptr == pQuery) {
return; return;
} }
...@@ -303,10 +309,10 @@ class ParserTestBaseImpl { ...@@ -303,10 +309,10 @@ class ParserTestBaseImpl {
setParseContext(sql, &cxt); setParseContext(sql, &cxt);
if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) { if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParseInsertSql(&cxt, query.get(), nullptr); doParseInsertSql(&cxt, query.get(), nullptr);
} else { } else {
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParse(&cxt, query.get()); doParse(&cxt, query.get());
SQuery* pQuery = *(query.get()); SQuery* pQuery = *(query.get());
...@@ -335,7 +341,7 @@ class ParserTestBaseImpl { ...@@ -335,7 +341,7 @@ class ParserTestBaseImpl {
SParseContext cxt = {0}; SParseContext cxt = {0};
setParseContext(sql, &cxt); setParseContext(sql, &cxt);
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParseSql(&cxt, query.get()); doParseSql(&cxt, query.get());
SQuery* pQuery = *(query.get()); SQuery* pQuery = *(query.get());
...@@ -354,26 +360,26 @@ class ParserTestBaseImpl { ...@@ -354,26 +360,26 @@ class ParserTestBaseImpl {
void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) {
reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL);
try { try {
SParseContext cxt = {0}; unique_ptr<SParseContext, function<void(SParseContext*)> > cxt(new SParseContext(), destoryParseContext);
setParseContext(sql, &cxt, true); setParseContext(sql, cxt.get(), true);
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
bool request = true; bool request = true;
unique_ptr<SParseMetaCache, function<void(SParseMetaCache*)> > metaCache( unique_ptr<SParseMetaCache, function<void(SParseMetaCache*)> > metaCache(
new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request)));
bool isInsertValues = qIsInsertValuesSql(cxt.pSql, cxt.sqlLen); bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen);
if (isInsertValues) { if (isInsertValues) {
doParseInsertSyntax(&cxt, query.get(), metaCache.get()); doParseInsertSyntax(cxt.get(), query.get(), metaCache.get());
} else { } else {
doParse(&cxt, query.get()); doParse(cxt.get(), query.get());
doCollectMetaKey(&cxt, *(query.get()), metaCache.get()); doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get());
} }
SQuery* pQuery = *(query.get()); SQuery* pQuery = *(query.get());
unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(), unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(),
MockCatalogService::destoryCatalogReq); MockCatalogService::destoryCatalogReq);
doBuildCatalogReq(&cxt, metaCache.get(), catalogReq.get()); doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get());
string err; string err;
thread t1([&]() { thread t1([&]() {
...@@ -386,13 +392,13 @@ class ParserTestBaseImpl { ...@@ -386,13 +392,13 @@ class ParserTestBaseImpl {
doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues); doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues);
if (isInsertValues) { if (isInsertValues) {
doParseInsertSql(&cxt, query.get(), metaCache.get()); doParseInsertSql(cxt.get(), query.get(), metaCache.get());
} else { } else {
doAuthenticate(&cxt, pQuery, metaCache.get()); doAuthenticate(cxt.get(), pQuery, metaCache.get());
doTranslate(&cxt, pQuery, metaCache.get()); doTranslate(cxt.get(), pQuery, metaCache.get());
doCalculateConstant(&cxt, pQuery); doCalculateConstant(cxt.get(), pQuery);
} }
} catch (const TerminateFlag& e) { } catch (const TerminateFlag& e) {
// success and terminate // success and terminate
...@@ -423,13 +429,13 @@ class ParserTestBaseImpl { ...@@ -423,13 +429,13 @@ class ParserTestBaseImpl {
void runAsyncApis(const string& sql, int32_t expect, ParserStage checkStage) { void runAsyncApis(const string& sql, int32_t expect, ParserStage checkStage) {
reset(expect, checkStage, TEST_INTERFACE_ASYNC_API); reset(expect, checkStage, TEST_INTERFACE_ASYNC_API);
try { try {
SParseContext cxt = {0}; unique_ptr<SParseContext, function<void(SParseContext*)> > cxt(new SParseContext(), destoryParseContext);
setParseContext(sql, &cxt); setParseContext(sql, cxt.get());
unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(), unique_ptr<SCatalogReq, void (*)(SCatalogReq*)> catalogReq(new SCatalogReq(),
MockCatalogService::destoryCatalogReq); MockCatalogService::destoryCatalogReq);
unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), _destroyQuery); unique_ptr<SQuery*, void (*)(SQuery**)> query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery);
doParseSqlSyntax(&cxt, query.get(), catalogReq.get()); doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get());
SQuery* pQuery = *(query.get()); SQuery* pQuery = *(query.get());
string err; string err;
...@@ -438,7 +444,7 @@ class ParserTestBaseImpl { ...@@ -438,7 +444,7 @@ class ParserTestBaseImpl {
unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData); unique_ptr<SMetaData, void (*)(SMetaData*)> metaData(new SMetaData(), MockCatalogService::destoryMetaData);
doGetAllMeta(catalogReq.get(), metaData.get()); doGetAllMeta(catalogReq.get(), metaData.get());
doAnalyseSqlSemantic(&cxt, catalogReq.get(), metaData.get(), pQuery); doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery);
} catch (const TerminateFlag& e) { } catch (const TerminateFlag& e) {
// success and terminate // success and terminate
} catch (const runtime_error& e) { } catch (const runtime_error& e) {
......
...@@ -1007,6 +1007,7 @@ static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSub ...@@ -1007,6 +1007,7 @@ static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSub
code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort); code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
nodesDestroyNode((SNode*)pScan);
code = nodesListMakeStrictAppend(&pSubplan->pChildren, code = nodesListMakeStrictAppend(&pSubplan->pChildren,
(SNode*)splCreateScanSubplan(pCxt, pMergeScan, SPLIT_FLAG_STABLE_SPLIT)); (SNode*)splCreateScanSubplan(pCxt, pMergeScan, SPLIT_FLAG_STABLE_SPLIT));
} }
......
...@@ -33,7 +33,10 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo ...@@ -33,7 +33,10 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
SLogicSubplan* pLogicSubplan = NULL; SLogicSubplan* pLogicSubplan = NULL;
SQueryLogicPlan* pLogicPlan = NULL; SQueryLogicPlan* pLogicPlan = NULL;
int32_t code = createLogicPlan(pCxt, &pLogicSubplan); int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
code = createLogicPlan(pCxt, &pLogicSubplan);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = optimizeLogicPlan(pCxt, pLogicSubplan); code = optimizeLogicPlan(pCxt, pLogicSubplan);
} }
...@@ -49,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo ...@@ -49,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
dumpQueryPlan(*pPlan); dumpQueryPlan(*pPlan);
} }
nodesReleaseAllocator(pCxt->allocatorId);
nodesDestroyNode((SNode*)pLogicSubplan); nodesDestroyNode((SNode*)pLogicSubplan);
nodesDestroyNode((SNode*)pLogicPlan); nodesDestroyNode((SNode*)pLogicPlan);
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include "mockCatalog.h" #include "mockCatalog.h"
#include "parser.h" #include "parser.h"
#include "planTestUtil.h" #include "planTestUtil.h"
#include "tglobal.h"
class PlannerEnv : public testing::Environment { class PlannerEnv : public testing::Environment {
public: public:
...@@ -30,6 +31,8 @@ class PlannerEnv : public testing::Environment { ...@@ -30,6 +31,8 @@ class PlannerEnv : public testing::Environment {
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog(TD_TMP_DIR_PATH "td"); initLog(TD_TMP_DIR_PATH "td");
initCfg();
nodesInitAllocatorSet();
} }
virtual void TearDown() { virtual void TearDown() {
...@@ -37,6 +40,7 @@ class PlannerEnv : public testing::Environment { ...@@ -37,6 +40,7 @@ class PlannerEnv : public testing::Environment {
qCleanupKeywordsTable(); qCleanupKeywordsTable();
fmFuncMgtDestroy(); fmFuncMgtDestroy();
taosCloseLog(); taosCloseLog();
nodesDestroyAllocatorSet();
} }
PlannerEnv() {} PlannerEnv() {}
...@@ -67,6 +71,8 @@ class PlannerEnv : public testing::Environment { ...@@ -67,6 +71,8 @@ class PlannerEnv : public testing::Environment {
std::cout << "failed to init log file" << std::endl; std::cout << "failed to init log file" << std::endl;
} }
} }
void initCfg() { tsQueryPlannerTrace = true; }
}; };
static void parseArg(int argc, char* argv[]) { static void parseArg(int argc, char* argv[]) {
...@@ -79,6 +85,7 @@ static void parseArg(int argc, char* argv[]) { ...@@ -79,6 +85,7 @@ static void parseArg(int argc, char* argv[]) {
{"limitSql", required_argument, NULL, 'i'}, {"limitSql", required_argument, NULL, 'i'},
{"log", required_argument, NULL, 'l'}, {"log", required_argument, NULL, 'l'},
{"queryPolicy", required_argument, NULL, 'q'}, {"queryPolicy", required_argument, NULL, 'q'},
{"useNodeAllocator", required_argument, NULL, 'a'},
{0, 0, 0, 0} {0, 0, 0, 0}
}; };
// clang-format on // clang-format on
...@@ -99,6 +106,9 @@ static void parseArg(int argc, char* argv[]) { ...@@ -99,6 +106,9 @@ static void parseArg(int argc, char* argv[]) {
case 'q': case 'q':
setQueryPolicy(optarg); setQueryPolicy(optarg);
break; break;
case 'a':
setUseNodeAllocator(optarg);
break;
default: default:
break; break;
} }
......
...@@ -41,6 +41,7 @@ using namespace testing; ...@@ -41,6 +41,7 @@ using namespace testing;
enum DumpModule { enum DumpModule {
DUMP_MODULE_NOTHING = 1, DUMP_MODULE_NOTHING = 1,
DUMP_MODULE_SQL,
DUMP_MODULE_PARSER, DUMP_MODULE_PARSER,
DUMP_MODULE_LOGIC, DUMP_MODULE_LOGIC,
DUMP_MODULE_OPTIMIZED, DUMP_MODULE_OPTIMIZED,
...@@ -56,10 +57,13 @@ int32_t g_skipSql = 0; ...@@ -56,10 +57,13 @@ int32_t g_skipSql = 0;
int32_t g_limitSql = 0; int32_t g_limitSql = 0;
int32_t g_logLevel = 131; int32_t g_logLevel = 131;
int32_t g_queryPolicy = QUERY_POLICY_VNODE; int32_t g_queryPolicy = QUERY_POLICY_VNODE;
bool g_useNodeAllocator = false;
void setDumpModule(const char* pModule) { void setDumpModule(const char* pModule) {
if (NULL == pModule) { if (NULL == pModule) {
g_dumpModule = DUMP_MODULE_ALL; g_dumpModule = DUMP_MODULE_ALL;
} else if (0 == strncasecmp(pModule, "sql", strlen(pModule))) {
g_dumpModule = DUMP_MODULE_SQL;
} else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) {
g_dumpModule = DUMP_MODULE_PARSER; g_dumpModule = DUMP_MODULE_PARSER;
} else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) {
...@@ -79,10 +83,11 @@ void setDumpModule(const char* pModule) { ...@@ -79,10 +83,11 @@ void setDumpModule(const char* pModule) {
} }
} }
void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } void setSkipSqlNum(const char* pArg) { g_skipSql = stoi(pArg); }
void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); } void setLimitSqlNum(const char* pArg) { g_limitSql = stoi(pArg); }
void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } void setLogLevel(const char* pArg) { g_logLevel = stoi(pArg); }
void setQueryPolicy(const char* pQueryPolicy) { g_queryPolicy = stoi(pQueryPolicy); } void setQueryPolicy(const char* pArg) { g_queryPolicy = stoi(pArg); }
void setUseNodeAllocator(const char* pArg) { g_useNodeAllocator = stoi(pArg); }
int32_t getLogLevel() { return g_logLevel; } int32_t getLogLevel() { return g_logLevel; }
...@@ -124,6 +129,12 @@ class PlannerTestBaseImpl { ...@@ -124,6 +129,12 @@ class PlannerTestBaseImpl {
} }
void runImpl(const string& sql, int32_t queryPolicy) { void runImpl(const string& sql, int32_t queryPolicy) {
int64_t allocatorId = 0;
if (g_useNodeAllocator) {
nodesCreateAllocator(sqlNo_, 32 * 1024, &allocatorId);
nodesAcquireAllocator(allocatorId);
}
reset(); reset();
tsQueryPolicy = queryPolicy; tsQueryPolicy = queryPolicy;
try { try {
...@@ -155,8 +166,13 @@ class PlannerTestBaseImpl { ...@@ -155,8 +166,13 @@ class PlannerTestBaseImpl {
dump(g_dumpModule); dump(g_dumpModule);
} catch (...) { } catch (...) {
dump(DUMP_MODULE_ALL); dump(DUMP_MODULE_ALL);
nodesReleaseAllocator(allocatorId);
nodesDestroyAllocator(allocatorId);
throw; throw;
} }
nodesReleaseAllocator(allocatorId);
nodesDestroyAllocator(allocatorId);
} }
void prepare(const string& sql) { void prepare(const string& sql) {
...@@ -216,6 +232,8 @@ class PlannerTestBaseImpl { ...@@ -216,6 +232,8 @@ class PlannerTestBaseImpl {
doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan);
unique_ptr<SQueryPlan, void (*)(SQueryPlan*)> plan(pPlan, (void (*)(SQueryPlan*))nodesDestroyNode); unique_ptr<SQueryPlan, void (*)(SQueryPlan*)> plan(pPlan, (void (*)(SQueryPlan*))nodesDestroyNode);
checkPlanMsg((SNode*)pPlan);
dump(g_dumpModule); dump(g_dumpModule);
} catch (...) { } catch (...) {
dump(DUMP_MODULE_ALL); dump(DUMP_MODULE_ALL);
...@@ -252,7 +270,6 @@ class PlannerTestBaseImpl { ...@@ -252,7 +270,6 @@ class PlannerTestBaseImpl {
string splitLogicPlan_; string splitLogicPlan_;
string scaledLogicPlan_; string scaledLogicPlan_;
string physiPlan_; string physiPlan_;
string physiPlanMsg_;
vector<string> physiSubplans_; vector<string> physiSubplans_;
}; };
...@@ -276,17 +293,16 @@ class PlannerTestBaseImpl { ...@@ -276,17 +293,16 @@ class PlannerTestBaseImpl {
res_.splitLogicPlan_.clear(); res_.splitLogicPlan_.clear();
res_.scaledLogicPlan_.clear(); res_.scaledLogicPlan_.clear();
res_.physiPlan_.clear(); res_.physiPlan_.clear();
res_.physiPlanMsg_.clear();
res_.physiSubplans_.clear(); res_.physiSubplans_.clear();
} }
void dump(DumpModule module) { void dump(DumpModule module) {
cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl;
if (DUMP_MODULE_NOTHING == module) { if (DUMP_MODULE_NOTHING == module) {
return; return;
} }
cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl;
if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) {
if (res_.prepareAst_.empty()) { if (res_.prepareAst_.empty()) {
cout << "+++++++++++++++++++++syntax tree : " << endl; cout << "+++++++++++++++++++++syntax tree : " << endl;
...@@ -411,8 +427,6 @@ class PlannerTestBaseImpl { ...@@ -411,8 +427,6 @@ class PlannerTestBaseImpl {
SNode* pSubplan; SNode* pSubplan;
FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); } FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); }
} }
res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan));
cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl;
} }
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
...@@ -451,27 +465,16 @@ class PlannerTestBaseImpl { ...@@ -451,27 +465,16 @@ class PlannerTestBaseImpl {
string toString(const SNode* pRoot) { string toString(const SNode* pRoot) {
char* pStr = NULL; char* pStr = NULL;
int32_t len = 0; int32_t len = 0;
auto start = chrono::steady_clock::now();
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) {
cout << "nodesNodeToString: "
<< chrono::duration_cast<chrono::microseconds>(chrono::steady_clock::now() - start).count() << "us" << endl;
}
string str(pStr); string str(pStr);
taosMemoryFreeClear(pStr); taosMemoryFreeClear(pStr);
return str; return str;
} }
string toMsg(const SNode* pRoot) { void checkPlanMsg(const SNode* pRoot) {
char* pStr = NULL; char* pStr = NULL;
int32_t len = 0; int32_t len = 0;
auto start = chrono::steady_clock::now();
DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len) DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len)
cout << "nodesNodeToMsg: "
<< chrono::duration_cast<chrono::microseconds>(chrono::steady_clock::now() - start).count() << "us" << endl;
string copyStr(pStr, len); string copyStr(pStr, len);
SNode* pNode = NULL; SNode* pNode = NULL;
...@@ -491,9 +494,7 @@ class PlannerTestBaseImpl { ...@@ -491,9 +494,7 @@ class PlannerTestBaseImpl {
nodesDestroyNode(pNode); nodesDestroyNode(pNode);
taosMemoryFreeClear(pNewStr); taosMemoryFreeClear(pNewStr);
string str(pStr, len);
taosMemoryFreeClear(pStr); taosMemoryFreeClear(pStr);
return str;
} }
caseEnv caseEnv_; caseEnv caseEnv_;
......
...@@ -41,11 +41,12 @@ class PlannerTestBase : public testing::Test { ...@@ -41,11 +41,12 @@ class PlannerTestBase : public testing::Test {
std::unique_ptr<PlannerTestBaseImpl> impl_; std::unique_ptr<PlannerTestBaseImpl> impl_;
}; };
extern void setDumpModule(const char* pModule); extern void setDumpModule(const char* pArg);
extern void setSkipSqlNum(const char* pNum); extern void setSkipSqlNum(const char* pArg);
extern void setLimitSqlNum(const char* pNum); extern void setLimitSqlNum(const char* pArg);
extern void setLogLevel(const char* pLogLevel); extern void setLogLevel(const char* pArg);
extern void setQueryPolicy(const char* pQueryPolicy); extern void setQueryPolicy(const char* pArg);
extern void setUseNodeAllocator(const char* pArg);
extern int32_t getLogLevel(); extern int32_t getLogLevel();
#endif // PLAN_TEST_UTIL_H #endif // PLAN_TEST_UTIL_H
...@@ -847,7 +847,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { ...@@ -847,7 +847,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
memcpy(res->datum.p, output.columnData->pData, len); memcpy(res->datum.p, output.columnData->pData, len);
} else if (IS_VAR_DATA_TYPE(type)) { } else if (IS_VAR_DATA_TYPE(type)) {
//res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); //res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1);
res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData), 1); res->datum.p = taosMemoryCalloc(varDataTLen(output.columnData->pData) + 1, 1);
res->node.resType.bytes = varDataTLen(output.columnData->pData); res->node.resType.bytes = varDataTLen(output.columnData->pData);
memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData));
} else { } else {
......
...@@ -254,7 +254,8 @@ typedef struct SSchJob { ...@@ -254,7 +254,8 @@ typedef struct SSchJob {
SRequestConnInfo conn; SRequestConnInfo conn;
SArray *nodeList; // qnode/vnode list, SArray<SQueryNodeLoad> SArray *nodeList; // qnode/vnode list, SArray<SQueryNodeLoad>
SArray *levels; // starting from 0. SArray<SSchLevel> SArray *levels; // starting from 0. SArray<SSchLevel>
SQueryPlan *pDag; SQueryPlan *pDag;
int64_t allocatorRefId;
SArray *dataSrcTasks; // SArray<SQueryTask*> SArray *dataSrcTasks; // SArray<SQueryTask*>
int32_t levelIdx; int32_t levelIdx;
......
...@@ -673,6 +673,7 @@ void schFreeJobImpl(void *job) { ...@@ -673,6 +673,7 @@ void schFreeJobImpl(void *job) {
destroyQueryExecRes(&pJob->execRes); destroyQueryExecRes(&pJob->execRes);
qDestroyQueryPlan(pJob->pDag); qDestroyQueryPlan(pJob->pDag);
nodesReleaseAllocatorWeakRef(pJob->allocatorRefId);
taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->userRes.execRes);
taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->fetchRes);
...@@ -724,6 +725,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { ...@@ -724,6 +725,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
pJob->sql = strdup(pReq->sql); pJob->sql = strdup(pReq->sql);
} }
pJob->pDag = pReq->pDag; pJob->pDag = pReq->pDag;
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillFp = pReq->chkKillFp;
pJob->chkKillParam = pReq->chkKillParam; pJob->chkKillParam = pReq->chkKillParam;
pJob->userRes.execFp = pReq->execFp; pJob->userRes.execFp = pReq->execFp;
......
...@@ -182,7 +182,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S ...@@ -182,7 +182,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
pReq->upstreamTaskId); pReq->upstreamTaskId);
streamTaskEnqueue(pTask, pReq, pRsp); streamTaskEnqueue(pTask, pReq, pRsp);
tFreeStreamDispatchReq(pReq); tDeleteStreamDispatchReq(pReq);
if (exec) { if (exec) {
if (streamTryExec(pTask) < 0) { if (streamTryExec(pTask) < 0) {
......
...@@ -179,5 +179,15 @@ void streamFreeQitem(SStreamQueueItem* data) { ...@@ -179,5 +179,15 @@ void streamFreeQitem(SStreamQueueItem* data) {
taosArrayDestroy(pMerge->reqs); taosArrayDestroy(pMerge->reqs);
taosArrayDestroy(pMerge->dataRefs); taosArrayDestroy(pMerge->dataRefs);
taosFreeQitem(pMerge); taosFreeQitem(pMerge);
} else if (type == STREAM_INPUT__REF_DATA_BLOCK) {
SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data;
int32_t ref = atomic_sub_fetch_32(pRefBlock->dataRef, 1);
ASSERT(ref >= 0);
if (ref == 0) {
blockDataDestroy(pRefBlock->pBlock);
taosMemoryFree(pRefBlock->dataRef);
}
taosFreeQitem(pRefBlock);
} }
} }
...@@ -62,7 +62,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { ...@@ -62,7 +62,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
return 0; return 0;
} }
void tFreeStreamDispatchReq(SStreamDispatchReq* pReq) { void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq) {
taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroyP(pReq->data, taosMemoryFree);
taosArrayDestroy(pReq->dataLen); taosArrayDestroy(pReq->dataLen);
} }
...@@ -95,7 +95,10 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { ...@@ -95,7 +95,10 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) {
return 0; return 0;
} }
void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq) { taosMemoryFree(pReq->pRetrieve); }
int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) { int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) {
int32_t code = -1;
SRetrieveTableRsp* pRetrieve = NULL; SRetrieveTableRsp* pRetrieve = NULL;
void* buf = NULL; void* buf = NULL;
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
...@@ -143,7 +146,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ...@@ -143,7 +146,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
buf = rpcMallocCont(sizeof(SMsgHead) + len); buf = rpcMallocCont(sizeof(SMsgHead) + len);
if (buf == NULL) { if (buf == NULL) {
goto FAIL; goto CLEAR;
} }
((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId); ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId);
...@@ -151,6 +154,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ...@@ -151,6 +154,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeStreamRetrieveReq(&encoder, &req); tEncodeStreamRetrieveReq(&encoder, &req);
tEncoderClear(&encoder);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.code = 0, .code = 0,
...@@ -161,17 +165,18 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ...@@ -161,17 +165,18 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
ASSERT(0); ASSERT(0);
return -1; goto CLEAR;
} }
buf = NULL;
qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId, qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId,
pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId);
} }
return 0; code = 0;
FAIL: CLEAR:
if (pRetrieve) taosMemoryFree(pRetrieve); taosMemoryFree(pRetrieve);
if (buf) taosMemoryFree(buf); rpcFreeCont(buf);
return -1; return code;
} }
static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
......
...@@ -38,6 +38,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* ...@@ -38,6 +38,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
SArray* blocks = pMerged->reqs; SArray* blocks = pMerged->reqs;
qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size);
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT);
} else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) {
const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data;
qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK);
} else { } else {
ASSERT(0); ASSERT(0);
} }
......
...@@ -69,15 +69,26 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { ...@@ -69,15 +69,26 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
if (agree) { if (agree) {
// term // term
SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); SSyncRaftEntry* pEntry = NULL;
ASSERT(pEntry != NULL); SLRUCache* pCache = pSyncNode->pLogStore->pCache;
LRUHandle* h = taosLRUCacheLookup(pCache, &index, sizeof(index));
if (h) {
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
ASSERT(pEntry != NULL);
}
// cannot commit, even if quorum agree. need check term! // cannot commit, even if quorum agree. need check term!
if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) {
// update commit index // update commit index
newCommitIndex = index; newCommitIndex = index;
syncEntryDestory(pEntry); if (h) {
taosLRUCacheRelease(pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
break; break;
} else { } else {
do { do {
...@@ -88,7 +99,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { ...@@ -88,7 +99,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
} while (0); } while (0);
} }
syncEntryDestory(pEntry); if (h) {
taosLRUCacheRelease(pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
} }
} }
......
...@@ -2581,6 +2581,20 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { ...@@ -2581,6 +2581,20 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) {
return ret; return ret;
} }
static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); }
static int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h) {
int code = 0;
int entryLen = sizeof(*pEntry) + pEntry->dataLen;
LRUStatus status = taosLRUCacheInsert(pLogStore->pCache, &pEntry->index, sizeof(pEntry->index), pEntry, entryLen,
deleteCacheEntry, h, TAOS_LRU_PRIORITY_LOW);
if (status != TAOS_LRU_STATUS_OK) {
code = -1;
}
return code;
}
static int32_t syncNodeAppendNoop(SSyncNode* ths) { static int32_t syncNodeAppendNoop(SSyncNode* ths) {
int32_t ret = 0; int32_t ret = 0;
...@@ -2589,13 +2603,21 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { ...@@ -2589,13 +2603,21 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId);
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);
LRUHandle* h = NULL;
syncCacheEntry(ths->pLogStore, pEntry, &h);
if (ths->state == TAOS_SYNC_STATE_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER) {
int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
ASSERT(code == 0); ASSERT(code == 0);
syncNodeReplicate(ths, false); syncNodeReplicate(ths, false);
} }
syncEntryDestory(pEntry); if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
return ret; return ret;
} }
...@@ -2654,6 +2676,9 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI ...@@ -2654,6 +2676,9 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index);
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);
LRUHandle* h = NULL;
syncCacheEntry(ths->pLogStore, pEntry, &h);
if (ths->state == TAOS_SYNC_STATE_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER) {
// append entry // append entry
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
...@@ -2685,7 +2710,12 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI ...@@ -2685,7 +2710,12 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
} }
} }
syncEntryDestory(pEntry); if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
return ret; return ret;
} }
...@@ -2973,9 +3003,15 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ...@@ -2973,9 +3003,15 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
for (SyncIndex i = beginIndex; i <= endIndex; ++i) { for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
if (i != SYNC_INDEX_INVALID) { if (i != SYNC_INDEX_INVALID) {
SSyncRaftEntry* pEntry; SSyncRaftEntry* pEntry;
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); SLRUCache* pCache = ths->pLogStore->pCache;
ASSERT(code == 0); LRUHandle* h = taosLRUCacheLookup(pCache, &i, sizeof(i));
ASSERT(pEntry != NULL); if (h) {
pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h);
} else {
code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
ASSERT(code == 0);
ASSERT(pEntry != NULL);
}
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pEntry, &rpcMsg); syncEntry2OriginalRpc(pEntry, &rpcMsg);
...@@ -3058,7 +3094,11 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ...@@ -3058,7 +3094,11 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex,
} }
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);
syncEntryDestory(pEntry); if (h) {
taosLRUCacheRelease(pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
} }
} }
} }
......
...@@ -53,6 +53,15 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { ...@@ -53,6 +53,15 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore));
ASSERT(pLogStore != NULL); ASSERT(pLogStore != NULL);
pLogStore->pCache = taosLRUCacheInit(10 * 1024 * 1024, 1, .5);
if (pLogStore->pCache == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
taosMemoryFree(pLogStore);
return NULL;
}
taosLRUCacheSetStrictCapacity(pLogStore->pCache, false);
pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData));
ASSERT(pLogStore->data != NULL); ASSERT(pLogStore->data != NULL);
...@@ -102,6 +111,10 @@ void logStoreDestory(SSyncLogStore* pLogStore) { ...@@ -102,6 +111,10 @@ void logStoreDestory(SSyncLogStore* pLogStore) {
taosThreadMutexDestroy(&(pData->mutex)); taosThreadMutexDestroy(&(pData->mutex));
taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore->data);
taosLRUCacheEraseUnrefEntries(pLogStore->pCache);
taosLRUCacheCleanup(pLogStore->pCache);
taosMemoryFree(pLogStore); taosMemoryFree(pLogStore);
} }
} }
...@@ -243,7 +256,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr ...@@ -243,7 +256,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
int32_t code; int32_t code = 0;
*ppEntry = NULL; *ppEntry = NULL;
...@@ -257,6 +270,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, ...@@ -257,6 +270,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
taosThreadMutexLock(&(pData->mutex)); taosThreadMutexLock(&(pData->mutex));
code = walReadVer(pWalHandle, index); code = walReadVer(pWalHandle, index);
// code = walReadVerCached(pWalHandle, index);
if (code != 0) { if (code != 0) {
int32_t err = terrno; int32_t err = terrno;
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
...@@ -412,6 +426,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { ...@@ -412,6 +426,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
ASSERT(pWalHandle != NULL); ASSERT(pWalHandle != NULL);
int32_t code = walReadVer(pWalHandle, index); int32_t code = walReadVer(pWalHandle, index);
// int32_t code = walReadVerCached(pWalHandle, index);
if (code != 0) { if (code != 0) {
int32_t err = terrno; int32_t err = terrno;
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
......
...@@ -80,6 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) ...@@ -80,6 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
ASSERT(xFree); ASSERT(xFree);
for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) {
tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage);
tdbOsFree(pPage->apOvfl[iOvfl]); tdbOsFree(pPage->apOvfl[iOvfl]);
} }
...@@ -152,7 +153,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl ...@@ -152,7 +153,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
pNewCell = (SCell *)tdbOsMalloc(szCell); pNewCell = (SCell *)tdbOsMalloc(szCell);
memcpy(pNewCell, pCell, szCell); memcpy(pNewCell, pCell, szCell);
tdbDebug("tdbPage/new ovfl cell: %p", pNewCell); tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage);
pPage->apOvfl[iOvfl] = pNewCell; pPage->apOvfl[iOvfl] = pNewCell;
pPage->aiOvfl[iOvfl] = idx; pPage->aiOvfl[iOvfl] = idx;
...@@ -202,7 +203,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { ...@@ -202,7 +203,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
if (pPage->aiOvfl[iOvfl] == idx) { if (pPage->aiOvfl[iOvfl] == idx) {
// remove the over flow cell // remove the over flow cell
tdbOsFree(pPage->apOvfl[iOvfl]); tdbOsFree(pPage->apOvfl[iOvfl]);
tdbDebug("tdbPage/free ovfl cell: %p", pPage->apOvfl[iOvfl]); tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]);
for (; (++iOvfl) < pPage->nOverflow;) { for (; (++iOvfl) < pPage->nOverflow;) {
pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1;
pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl];
...@@ -255,6 +256,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { ...@@ -255,6 +256,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
pNewCell = (SCell *)tdbOsMalloc(szCell); pNewCell = (SCell *)tdbOsMalloc(szCell);
memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
} }
pToPage->apOvfl[iOvfl] = pNewCell; pToPage->apOvfl[iOvfl] = pNewCell;
......
...@@ -116,7 +116,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { ...@@ -116,7 +116,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
} }
#endif #endif
} }
// TODO truncate file
if (found == NULL) { if (found == NULL) {
// file corrupted, no complete log // file corrupted, no complete log
...@@ -125,8 +124,20 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { ...@@ -125,8 +124,20 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
// truncate file
SWalCkHead* lastEntry = (SWalCkHead*)found; SWalCkHead* lastEntry = (SWalCkHead*)found;
int64_t retVer = lastEntry->head.version; int64_t retVer = lastEntry->head.version;
int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf);
int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen;
if (lastEntryEndOffset != fileSize) {
wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
fileSize);
taosFtruncateFile(pFile, lastEntryEndOffset);
((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset;
pWal->totSize -= (fileSize - lastEntryEndOffset);
}
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(buf); taosMemoryFree(buf);
...@@ -226,16 +237,92 @@ int walCheckAndRepairMeta(SWal* pWal) { ...@@ -226,16 +237,92 @@ int walCheckAndRepairMeta(SWal* pWal) {
} }
} }
// TODO: set fileSize and lastVer if necessary
return 0; return 0;
} }
int walCheckAndRepairIdx(SWal* pWal) { int walCheckAndRepairIdx(SWal* pWal) {
// TODO: iterate all log files int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
// if idx not found, scan log and write idx for (int32_t i = 0; i < sz; i++) {
// if found, check complete by first and last entry of each idx file SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i);
// if idx incomplete, binary search last valid entry, and then build other part
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fsize;
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE);
if (pIdxFile == NULL) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr());
return -1;
}
taosFStatFile(pIdxFile, &fsize, NULL);
if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
taosCloseFile(&pIdxFile);
continue;
}
int32_t left = fsize % sizeof(SWalIdxEntry);
int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END);
if (left != 0) {
taosFtruncateFile(pIdxFile, offset);
wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr,
offset, fsize);
}
offset -= sizeof(SWalIdxEntry);
SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer};
while (1) {
if (offset < 0) {
taosLSeekFile(pIdxFile, 0, SEEK_SET);
taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
break;
}
taosLSeekFile(pIdxFile, offset, SEEK_SET);
int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) {
taosFtruncateFile(pIdxFile, offset);
wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld",
pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset);
offset -= sizeof(SWalIdxEntry);
} else {
break;
}
}
if (idxEntry.ver < pFileInfo->lastVer) {
char fLogNameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ);
if (pLogFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr());
return -1;
}
while (idxEntry.ver < pFileInfo->lastVer) {
taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET);
SWalCkHead ckHead;
taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead));
if (idxEntry.ver != ckHead.head.version) {
// todo truncate this idx also
taosCloseFile(&pLogFile);
wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId,
idxEntry.offset, idxEntry.ver, ckHead.head.version);
return -1;
}
idxEntry.ver = ckHead.head.version + 1;
idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen;
wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset);
taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry));
}
taosCloseFile(&pLogFile);
}
taosCloseFile(&pIdxFile);
}
return 0; return 0;
} }
......
...@@ -149,15 +149,21 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { ...@@ -149,15 +149,21 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
walLoadMeta(pWal); walLoadMeta(pWal);
if (walCheckAndRepairMeta(pWal) < 0) { if (walCheckAndRepairMeta(pWal) < 0) {
wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId);
taosHashCleanup(pWal->pRefHash); taosHashCleanup(pWal->pRefHash);
taosRemoveRef(tsWal.refSetId, pWal->refId); taosRemoveRef(tsWal.refSetId, pWal->refId);
taosThreadMutexDestroy(&pWal->mutex); taosThreadMutexDestroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet); taosArrayDestroy(pWal->fileInfoSet);
taosMemoryFree(pWal);
return NULL; return NULL;
} }
if (walCheckAndRepairIdx(pWal) < 0) { if (walCheckAndRepairIdx(pWal) < 0) {
wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId);
taosHashCleanup(pWal->pRefHash);
taosRemoveRef(tsWal.refSetId, pWal->refId);
taosThreadMutexDestroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet);
return NULL;
} }
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
......
...@@ -284,7 +284,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { ...@@ -284,7 +284,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
// all pages are referenced by user, try to allocate new space // all pages are referenced by user, try to allocate new space
if (pn == NULL) { if (pn == NULL) {
assert(0);
int32_t prev = pBuf->inMemPages; int32_t prev = pBuf->inMemPages;
// increase by 50% of previous mem pages // increase by 50% of previous mem pages
...@@ -304,7 +303,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { ...@@ -304,7 +303,6 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
bufPage = flushPageToDisk(pBuf, d); bufPage = flushPageToDisk(pBuf, d);
} }
ASSERT((bufPage != NULL) || terrno != TSDB_CODE_SUCCESS);
return bufPage; return bufPage;
} }
...@@ -377,12 +375,6 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { ...@@ -377,12 +375,6 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
char* availablePage = NULL; char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
availablePage = evacOneDataPage(pBuf); availablePage = evacOneDataPage(pBuf);
// Failed to allocate a new buffer page, and there is an error occurs.
if (availablePage == NULL) {
assert(0);
return NULL;
}
} }
SPageInfo* pi = NULL; SPageInfo* pi = NULL;
...@@ -652,4 +644,4 @@ void clearDiskbasedBuf(SDiskbasedBuf* pBuf) { ...@@ -652,4 +644,4 @@ void clearDiskbasedBuf(SDiskbasedBuf* pBuf) {
pBuf->totalBufSize = 0; pBuf->totalBufSize = 0;
pBuf->allocateId = -1; pBuf->allocateId = -1;
pBuf->fileSize = 0; pBuf->fileSize = 0;
} }
\ No newline at end of file
...@@ -6,23 +6,24 @@ pgrep taosd || taosd >> /dev/null 2>&1 & ...@@ -6,23 +6,24 @@ pgrep taosd || taosd >> /dev/null 2>&1 &
pgrep taosadapter || taosadapter >> /dev/null 2>&1 & pgrep taosadapter || taosadapter >> /dev/null 2>&1 &
cd ../../docs/examples/csharp cd ../../docs/examples/csharp
dotnet run --project connect.csproj #dotnet run --project connect.csproj
taos -s "drop database if exists power" #taos -s "drop database if exists power"
dotnet run --project sqlinsert.csproj #dotnet run --project sqlinsert.csproj
dotnet run --project query.csproj #dotnet run --project query.csproj
dotnet run --project asyncquery.csproj #dotnet run --project asyncquery.csproj
dotnet run --project subscribe.csproj #dotnet run --project subscribe.csproj
taos -s "drop topic if exists topic_example" #taos -s "drop topic if exists topic_example"
taos -s "drop database if exists power" #taos -s "drop database if exists power"
dotnet run --project stmtinsert.csproj #dotnet run --project stmtinsert.csproj
taos -s "drop database if exists test" #taos -s "drop database if exists test"
dotnet run --project influxdbline.csproj #dotnet run --project influxdbline.csproj
taos -s "drop database if exists test" #taos -s "drop database if exists test"
dotnet run --project optstelnet.csproj #dotnet run --project optstelnet.csproj
taos -s "drop database if exists test" #taos -s "drop database if exists test"
dotnet run --project optsjson.csproj #dotnet run --project optsjson.csproji
\ No newline at end of file echo "uncomment temporily"
...@@ -79,9 +79,11 @@ fi ...@@ -79,9 +79,11 @@ fi
ulimit -c unlimited ulimit -c unlimited
TMP_DIR=$WORKDIR/tmp TMP_DIR=$WORKDIR/tmp
SOURCEDIR=$WORKDIR/src
MOUNT_DIR="" MOUNT_DIR=""
packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz"
rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim
mkdir -p $SOURCEDIR
mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim
mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump
rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/* rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/*
...@@ -90,6 +92,11 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then ...@@ -90,6 +92,11 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then
echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/" echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/"
cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/
fi fi
if [ ! -f "${SOURCEDIR}/${packageName}" ]; then
wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName}
fi
MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir" MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir"
echo "$thread_no -> ${exec_dir}:$cmd" echo "$thread_no -> ${exec_dir}:$cmd"
coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname`
...@@ -97,6 +104,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` ...@@ -97,6 +104,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname`
docker run \ docker run \
-v $REP_MOUNT_PARAM \ -v $REP_MOUNT_PARAM \
-v $MOUNT_DIR \ -v $MOUNT_DIR \
-v ${SOURCEDIR}:/usr/local/src/ \
-v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \
-v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \
-v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \
......
from urllib.parse import uses_relative
import taos
import sys
import os
import time
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
from util.cluster import *
class TDTestCase:
def caseDescription(self):
'''
3.0 data compatibility test
case1: basedata version is 3.0.1.0
'''
return
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files or "taosd.exe" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def getCfgPath(self):
buildPath = self.getBuildPath()
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
cfgPath = buildPath + "/../sim/dnode1/cfg/"
else:
cfgPath = buildPath + "/../sim/dnode1/cfg/"
return cfgPath
def installTaosd(self,bPath,cPath):
# os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ")
# os.system(f" mv {bPath}/build {bPath}/build_bak ")
# os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ")
# os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ")
packagePath="/usr/local/src/"
packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz"
os.system(f"cd {packagePath} && tar xvf TDengine-server-3.0.1.0-Linux-x64.tar.gz && cd TDengine-server-3.0.1.0 && ./install.sh -e no " )
tdDnodes.stop(1)
print(f"start taosd: nohup taosd -c {cPath} & ")
os.system(f" nohup taosd -c {cPath} & " )
sleep(1)
def buildTaosd(self,bPath):
# os.system(f"mv {bPath}/build_bak {bPath}/build ")
os.system(f" cd {bPath} && make install ")
def run(self):
bPath=self.getBuildPath()
cPath=self.getCfgPath()
dbname = "test"
stb = f"{dbname}.meters"
self.installTaosd(bPath,cPath)
tableNumbers=100
recordNumbers1=100
recordNumbers2=1000
tdsqlF=tdCom.newTdSql()
print(tdsqlF)
tdsqlF.query(f"SELECT SERVER_VERSION();")
print(tdsqlF.query(f"SELECT SERVER_VERSION();"))
oldServerVersion=tdsqlF.queryResult[0][0]
tdLog.info(f"Base server version is {oldServerVersion}")
tdsqlF.query(f"SELECT CLIENT_VERSION();")
# the oldClientVersion can't be updated in the same python process,so the version is new compiled verison
oldClientVersion=tdsqlF.queryResult[0][0]
tdLog.info(f"Base client version is {oldClientVersion}")
tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}")
tdLog.info(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ")
os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ")
sleep(3)
# tdsqlF.query(f"select count(*) from {stb}")
# tdsqlF.checkData(0,0,tableNumbers*recordNumbers1)
os.system("pkill taosd")
sleep(1)
tdLog.printNoPrefix("==========step2:update new version ")
self.buildTaosd(bPath)
tdDnodes.start(1)
sleep(1)
tdsql=tdCom.newTdSql()
print(tdsql)
tdsql.query(f"SELECT SERVER_VERSION();")
nowServerVersion=tdsql.queryResult[0][0]
tdLog.info(f"New server version is {nowServerVersion}")
tdsql.query(f"SELECT CLIENT_VERSION();")
nowClientVersion=tdsql.queryResult[0][0]
tdLog.info(f"New client version is {nowClientVersion}")
tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}")
tdsql.query(f"select count(*) from {stb}")
tdsql.checkData(0,0,tableNumbers*recordNumbers1)
os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ")
tdsql.query(f"select count(*) from {stb}")
tdsql.checkData(0,0,tableNumbers*recordNumbers2)
tdsql=tdCom.newTdSql()
tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542")
tdsql.execute("drop database if exists db")
tdsql.execute("create database db")
tdsql.execute("use db")
tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);")
tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);")
tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);")
tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);")
tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);")
tdsql.query("select * from db.ct3")
tdsql.checkData(0,1,13)
tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);")
tdsql.query("select * from db.ct4")
tdsql.checkData(0,1,14)
tdsql.query("describe information_schema.ins_databases;")
qRows=tdsql.queryRows
for i in range(qRows) :
if tdsql.queryResult[i][0]=="retentions" :
return True
else:
return False
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())
此差异已折叠。
...@@ -17,6 +17,7 @@ python3 ./test.py -f 0-others/udf_cfg2.py ...@@ -17,6 +17,7 @@ python3 ./test.py -f 0-others/udf_cfg2.py
python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/sysinfo.py
python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/user_control.py
python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/fsync.py
python3 ./test.py -f 0-others/compatibility.py
python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
...@@ -35,6 +36,7 @@ python3 ./test.py -f 1-insert/mutil_stage.py ...@@ -35,6 +36,7 @@ python3 ./test.py -f 1-insert/mutil_stage.py
python3 ./test.py -f 1-insert/table_param_ttl.py -R python3 ./test.py -f 1-insert/table_param_ttl.py -R
python3 ./test.py -f 1-insert/update_data_muti_rows.py python3 ./test.py -f 1-insert/update_data_muti_rows.py
python3 ./test.py -f 1-insert/db_tb_name_check.py python3 ./test.py -f 1-insert/db_tb_name_check.py
python3 ./test.py -f 1-insert/database_pre_suf.py
python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/abs.py
python3 ./test.py -f 2-query/abs.py -R python3 ./test.py -f 2-query/abs.py -R
...@@ -156,8 +158,8 @@ python3 ./test.py -f 2-query/sin.py ...@@ -156,8 +158,8 @@ python3 ./test.py -f 2-query/sin.py
python3 ./test.py -f 2-query/sin.py -R python3 ./test.py -f 2-query/sin.py -R
python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py
python3 ./test.py -f 2-query/smaTest.py -R python3 ./test.py -f 2-query/smaTest.py -R
#python3 ./test.py -f 2-query/sml.py python3 ./test.py -f 2-query/sml.py
#python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/sml.py -R
python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/spread.py
python3 ./test.py -f 2-query/spread.py -R python3 ./test.py -f 2-query/spread.py -R
python3 ./test.py -f 2-query/sqrt.py python3 ./test.py -f 2-query/sqrt.py
...@@ -512,6 +514,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3 ...@@ -512,6 +514,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3
python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3
python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3
python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
#python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/sml.py -Q 3
python3 ./test.py -f 2-query/interp.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3
...@@ -143,7 +143,7 @@ void shellCheckConnectMode() { ...@@ -143,7 +143,7 @@ void shellCheckConnectMode() {
shell.args.port = 6041; shell.args.port = 6041;
} }
shell.args.dsn = taosMemoryCalloc(1, 1024); shell.args.dsn = taosMemoryCalloc(1, 1024);
snprintf(shell.args.dsn, 1024, "ws://%s:%d/rest/ws", snprintf(shell.args.dsn, 1024, "ws://%s:%d",
shell.args.host, shell.args.port); shell.args.host, shell.args.port);
} }
shell.args.cloud = false; shell.args.cloud = false;
......
...@@ -206,26 +206,31 @@ void shellRunSingleCommandWebsocketImp(char *command) { ...@@ -206,26 +206,31 @@ void shellRunSingleCommandWebsocketImp(char *command) {
printMode = true; // When output to a file, the switch does not work. printMode = true; // When output to a file, the switch does not work.
} }
if (!shell.ws_conn && shell_conn_ws_server(0)) {
return;
}
shell.stop_query = false; shell.stop_query = false;
st = taosGetTimestampUs(); WS_RES* res;
WS_RES* res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout); for (int reconnectNum = 0; reconnectNum < 2; reconnectNum++) {
int code = ws_errno(res); if (!shell.ws_conn && shell_conn_ws_server(0)) {
if (code != 0) { return;
et = taosGetTimestampUs();
fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6);
if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) {
fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n");
} else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) {
fprintf(stderr, "TDengine server is down, will try to reconnect\n");
shell.ws_conn = NULL;
} }
ws_free_result(res); st = taosGetTimestampUs();
return;
res = ws_query_timeout(shell.ws_conn, command, shell.args.timeout);
int code = ws_errno(res);
if (code != 0 && !shell.stop_query) {
et = taosGetTimestampUs();
fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6);
if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) {
fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n");
} else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) {
fprintf(stderr, "TDengine server is down, will try to reconnect\n");
shell.ws_conn = NULL;
}
ws_free_result(res);
if (reconnectNum == 0) continue;
return;
}
break;
} }
double execute_time = ws_take_timing(res)/1E6; double execute_time = ws_take_timing(res)/1E6;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册