提交 4cbaacb2 编写于 作者: Y yihaoDeng

Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

...@@ -42,12 +42,12 @@ def pre_test(){ ...@@ -42,12 +42,12 @@ def pre_test(){
killall -9 taosd ||echo "no taosd running" killall -9 taosd ||echo "no taosd running"
killall -9 gdb || echo "no gdb running" killall -9 gdb || echo "no gdb running"
cd ${WKC} cd ${WKC}
git checkout develop
git reset --hard HEAD~10 >/dev/null git reset --hard HEAD~10 >/dev/null
git checkout develop
git pull >/dev/null git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD git checkout -qf FETCH_HEAD
find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\; git clean -dfx
cd ${WK} cd ${WK}
git reset --hard HEAD~10 git reset --hard HEAD~10
git checkout develop git checkout develop
...@@ -55,7 +55,7 @@ def pre_test(){ ...@@ -55,7 +55,7 @@ def pre_test(){
cd ${WK} cd ${WK}
export TZ=Asia/Harbin export TZ=Asia/Harbin
date date
rm -rf ${WK}/debug git clean -dfx
mkdir debug mkdir debug
cd debug cd debug
cmake .. > /dev/null cmake .. > /dev/null
......
...@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) ...@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .) #INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED) IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.24-dist.jar DESTINATION connector/jdbc) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc)
ENDIF () ENDIF ()
ELSEIF (TD_DARWIN) ELSEIF (TD_DARWIN)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
......
...@@ -29,21 +29,21 @@ taos> DESCRIBE meters; ...@@ -29,21 +29,21 @@ taos> DESCRIBE meters;
## <a class="anchor" id="data-type"></a>支持的数据类型 ## <a class="anchor" id="data-type"></a>支持的数据类型
使用TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则: 使用 TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则:
- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如:```2017-08-12 18:25:58.128``` - 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```默认时间分辨率为毫秒。比如:```2017-08-12 18:25:58.128```
- 内部函数now是服务器的当前时间 - 内部函数 now 是客户端的当前时间
- 插入记录时,如果时间戳为now,插入数据时使用服务器当前时间 - 插入记录时,如果时间戳为 now,插入数据时使用提交这条记录的客户端的当前时间
- Epoch Time: 时间戳也可以是一个长整数,表示从1970-01-01 08:00:00.000开始的毫秒数 - Epoch Time:时间戳也可以是一个长整数,表示从 1970-01-01 08:00:00.000 开始的毫秒数
- 时间可以加减,比如 now-2h,表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n(自然月) 和 y(自然年)。 - 时间可以加减,比如 now-2h,表明查询时刻向前推 2 个小时(最近 2 小时)。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`,表示查询两周前整整一周的数据。在指定降频操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n(自然月) 和 y(自然年)。
TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMicrosecond就可支持微秒。 TDengine 缺省的时间戳是毫秒精度,但通过修改配置参数 enableMicrosecond 就可以支持微秒。
在TDengine中,普通表的数据模型中可使用以下10种数据类型。 在TDengine中,普通表的数据模型中可使用以下 10 种数据类型。
| | 类型 | Bytes | 说明 | | | 类型 | Bytes | 说明 |
| ---- | :-------: | ------ | ------------------------------------------------------------ | | ---- | :-------: | ------ | ------------------------------------------------------------ |
| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始,计时不能早于该时间。 | | 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始,计时不能早于该时间。(从 2.0.18 版本开始,已经去除了这一时间范围限制) |
| 2 | INT | 4 | 整型,范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | | 2 | INT | 4 | 整型,范围 [-2^31+1, 2^31-1], -2^31 用作 NULL |
| 3 | BIGINT | 8 | 长整型,范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 3 | BIGINT | 8 | 长整型,范围 [-2^63+1, 2^63-1], -2^63 用于 NULL |
| 4 | FLOAT | 4 | 浮点型,有效位数 6-7,范围 [-3.4E38, 3.4E38] | | 4 | FLOAT | 4 | 浮点型,有效位数 6-7,范围 [-3.4E38, 3.4E38] |
......
...@@ -120,7 +120,7 @@ function clean_service_on_systemd() { ...@@ -120,7 +120,7 @@ function clean_service_on_systemd() {
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/${nginx_service_name}.service" nginx_service_config="${service_config_dir}/${nginx_service_name}.service"
if [ -d ${bin_dir}/web ]; then if [ -d ${install_nginxd_dir} ]; then
if systemctl is-active --quiet ${nginx_service_name}; then if systemctl is-active --quiet ${nginx_service_name}; then
echo "Nginx for TDengine is running, stopping it..." echo "Nginx for TDengine is running, stopping it..."
${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null
......
...@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ...@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
POST_BUILD POST_BUILD
COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.24-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH}
COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMENT "build jdbc driver") COMMENT "build jdbc driver")
ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
<groupId>com.taosdata.jdbc</groupId> <groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId> <artifactId>taos-jdbcdriver</artifactId>
<version>2.0.24</version> <version>2.0.25</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>JDBCDriver</name> <name>JDBCDriver</name>
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.taosdata.jdbc</groupId> <groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId> <artifactId>taos-jdbcdriver</artifactId>
<version>2.0.24</version> <version>2.0.25</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>JDBCDriver</name> <name>JDBCDriver</name>
<url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url> <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
...@@ -62,6 +62,14 @@ ...@@ -62,6 +62,14 @@
</dependencies> </dependencies>
<build> <build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>**/*.md</exclude>
</excludes>
</resource>
</resources>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
......
...@@ -15,7 +15,7 @@ public abstract class AbstractParameterMetaData extends WrapperImpl implements P ...@@ -15,7 +15,7 @@ public abstract class AbstractParameterMetaData extends WrapperImpl implements P
@Override @Override
public int getParameterCount() throws SQLException { public int getParameterCount() throws SQLException {
return parameters.length; return parameters == null ? 0 : parameters.length;
} }
@Override @Override
......
...@@ -29,45 +29,25 @@ public class TSDBJNIConnector { ...@@ -29,45 +29,25 @@ public class TSDBJNIConnector {
private static volatile Boolean isInitialized = false; private static volatile Boolean isInitialized = false;
private TaosInfo taosInfo = TaosInfo.getInstance(); private TaosInfo taosInfo = TaosInfo.getInstance();
// Connection pointer used in C
private long taos = TSDBConstants.JNI_NULL_POINTER;
// result set status in current connection
private boolean isResultsetClosed = true;
private int affectedRows = -1;
static { static {
System.loadLibrary("taos"); System.loadLibrary("taos");
System.out.println("java.library.path:" + System.getProperty("java.library.path")); System.out.println("java.library.path:" + System.getProperty("java.library.path"));
} }
/**
* Connection pointer used in C
*/
private long taos = TSDBConstants.JNI_NULL_POINTER;
/**
* Result set pointer for the current connection
*/
// private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
/**
* result set status in current connection
*/
private boolean isResultsetClosed = true;
private int affectedRows = -1;
/**
* Whether the connection is closed
*/
public boolean isClosed() { public boolean isClosed() {
return this.taos == TSDBConstants.JNI_NULL_POINTER; return this.taos == TSDBConstants.JNI_NULL_POINTER;
} }
/**
* Returns the status of last result set in current connection
*/
public boolean isResultsetClosed() { public boolean isResultsetClosed() {
return this.isResultsetClosed; return this.isResultsetClosed;
} }
/**
* Initialize static variables in JNI to optimize performance
*/
public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning { public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning {
synchronized (isInitialized) { synchronized (isInitialized) {
if (!isInitialized) { if (!isInitialized) {
...@@ -93,11 +73,6 @@ public class TSDBJNIConnector { ...@@ -93,11 +73,6 @@ public class TSDBJNIConnector {
public static native String getTsCharset(); public static native String getTsCharset();
/**
* Get connection pointer
*
* @throws SQLException
*/
public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException {
if (this.taos != TSDBConstants.JNI_NULL_POINTER) { if (this.taos != TSDBConstants.JNI_NULL_POINTER) {
// this.closeConnectionImp(this.taos); // this.closeConnectionImp(this.taos);
...@@ -185,13 +160,6 @@ public class TSDBJNIConnector { ...@@ -185,13 +160,6 @@ public class TSDBJNIConnector {
private native String getErrMsgImp(long pSql); private native String getErrMsgImp(long pSql);
/**
* Get resultset pointer
* Each connection should have a single open result set at a time
*/
// public long getResultSet() {
// return taosResultSetPointer;
// }
private native long getResultSetImp(long connection, long pSql); private native long getResultSetImp(long connection, long pSql);
public boolean isUpdateQuery(long pSql) { public boolean isUpdateQuery(long pSql) {
...@@ -231,6 +199,7 @@ public class TSDBJNIConnector { ...@@ -231,6 +199,7 @@ public class TSDBJNIConnector {
// } // }
// return resCode; // return resCode;
// } // }
private native int freeResultSetImp(long connection, long result); private native int freeResultSetImp(long connection, long result);
/** /**
...@@ -323,8 +292,7 @@ public class TSDBJNIConnector { ...@@ -323,8 +292,7 @@ public class TSDBJNIConnector {
* Validate if a <I>create table</I> sql statement is correct without actually creating that table * Validate if a <I>create table</I> sql statement is correct without actually creating that table
*/ */
public boolean validateCreateTableSql(String sql) { public boolean validateCreateTableSql(String sql) {
long connection = taos; int res = validateCreateTableSqlImp(taos, sql.getBytes());
int res = validateCreateTableSqlImp(connection, sql.getBytes());
return res != 0 ? false : true; return res != 0 ? false : true;
} }
......
...@@ -84,9 +84,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { ...@@ -84,9 +84,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet {
case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP:
return new Timestamp(row.getDate(colIndex).getTime()); return new Timestamp(row.getDate(colIndex).getTime());
case TSDBConstants.TSDB_DATA_TYPE_BINARY: case TSDBConstants.TSDB_DATA_TYPE_BINARY:
return row.getString(colIndex).getBytes(); return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes();
case TSDBConstants.TSDB_DATA_TYPE_NCHAR: case TSDBConstants.TSDB_DATA_TYPE_NCHAR:
return row.getString(colIndex); return row.getString(colIndex) == null ? null : row.getString(colIndex);
default: default:
return row.get(colIndex); return row.get(colIndex);
} }
......
package com.taosdata.jdbc.utils;
public class OSUtils {
private static final String OS = System.getProperty("os.name").toLowerCase();
public static boolean isWindows() {
return OS.indexOf("win") >= 0;
}
public static boolean isMac() {
return OS.indexOf("mac") >= 0;
}
public static boolean isLinux() {
return OS.indexOf("nux") >= 0;
}
}
...@@ -17,7 +17,6 @@ public class DriverAutoloadTest { ...@@ -17,7 +17,6 @@ public class DriverAutoloadTest {
@Test @Test
public void testRestful() throws SQLException { public void testRestful() throws SQLException {
// Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
Connection conn = DriverManager.getConnection(url, properties); Connection conn = DriverManager.getConnection(url, properties);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
......
package com.taosdata.jdbc.cases;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.sql.*;
public class NullValueInResultSetForJdbcRestfulTest {
private static final String host = "127.0.0.1";
Connection conn;
@Test
public void test() {
try (Statement stmt = conn.createStatement()) {
ResultSet rs = stmt.executeQuery("select * from weather");
ResultSetMetaData meta = rs.getMetaData();
while (rs.next()) {
for (int i = 1; i <= meta.getColumnCount(); i++) {
Object value = rs.getObject(i);
System.out.print(meta.getColumnLabel(i) + ": " + value + "\t");
}
System.out.println();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
@Before
public void before() throws SQLException {
final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
conn = DriverManager.getConnection(url);
try (Statement stmt = conn.createStatement()) {
stmt.execute("drop database if exists test_null");
stmt.execute("create database if not exists test_null");
stmt.execute("use test_null");
stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))");
stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)");
stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)");
stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)");
stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)");
stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)");
stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)");
stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)");
stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')");
stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')");
} catch (SQLException e) {
e.printStackTrace();
}
}
@After
public void after() {
try {
if (conn != null)
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
package com.taosdata.jdbc.utils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class OSUtilsTest {
private String OS;
@Test
public void inWindows() {
Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows());
}
@Test
public void isMac() {
Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac());
}
@Test
public void isLinux() {
Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux());
}
@Before
public void before() {
OS = System.getProperty("os.name").toLowerCase();
}
}
\ No newline at end of file
...@@ -237,6 +237,18 @@ static int32_t dnodeInitStorage() { ...@@ -237,6 +237,18 @@ static int32_t dnodeInitStorage() {
return -1; return -1;
} }
TDIR *tdir = tfsOpendir("vnode_bak/.staging");
if (tfsReaddir(tdir) != NULL) {
dError("vnode_bak/.staging dir not empty, fix it first.");
tfsClosedir(tdir);
return -1;
}
if (tfsMkdir("vnode_bak/.staging") < 0) {
dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno));
return -1;
}
dnodeCheckDataDirOpenned(tsDnodeDir); dnodeCheckDataDirOpenned(tsDnodeDir);
dInfo("dnode storage is initialized at %s", tsDnodeDir); dInfo("dnode storage is initialized at %s", tsDnodeDir);
......
...@@ -205,7 +205,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { ...@@ -205,7 +205,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version); pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version);
pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite); pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite);
if (pWrite->code <= 0) pWrite->processedCount = 1; if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1);
if (pWrite->code > 0) pWrite->code = 0; if (pWrite->code > 0) pWrite->code = 0;
if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
......
...@@ -259,7 +259,7 @@ do { \ ...@@ -259,7 +259,7 @@ do { \
#define TSDB_MIN_TABLES 4 #define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 10000000 #define TSDB_MAX_TABLES 10000000
#define TSDB_DEFAULT_TABLES 1000000 #define TSDB_DEFAULT_TABLES 1000000
#define TSDB_TABLES_STEP 100 #define TSDB_TABLES_STEP 1000
#define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650 #define TSDB_MAX_DAYS_PER_FILE 3650
......
...@@ -79,9 +79,6 @@ typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); ...@@ -79,9 +79,6 @@ typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
// get file version // get file version
typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
// reset version
typedef int32_t (*FResetVersion)(int32_t vgId, uint64_t fver);
typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd);
typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd);
...@@ -99,7 +96,6 @@ typedef struct { ...@@ -99,7 +96,6 @@ typedef struct {
FStartSyncFile startSyncFileFp; FStartSyncFile startSyncFileFp;
FStopSyncFile stopSyncFileFp; FStopSyncFile stopSyncFileFp;
FGetVersion getVersionFp; FGetVersion getVersionFp;
FResetVersion resetVersionFp;
FSendFile sendFileFp; FSendFile sendFileFp;
FRecvFile recvFileFp; FRecvFile recvFileFp;
} SSyncInfo; } SSyncInfo;
......
此差异已折叠。
...@@ -117,7 +117,6 @@ typedef struct SSyncNode { ...@@ -117,7 +117,6 @@ typedef struct SSyncNode {
FStartSyncFile startSyncFileFp; FStartSyncFile startSyncFileFp;
FStopSyncFile stopSyncFileFp; FStopSyncFile stopSyncFileFp;
FGetVersion getVersionFp; FGetVersion getVersionFp;
FResetVersion resetVersionFp;
FSendFile sendFileFp; FSendFile sendFileFp;
FRecvFile recvFileFp; FRecvFile recvFileFp;
pthread_mutex_t mutex; pthread_mutex_t mutex;
......
...@@ -182,7 +182,6 @@ int64_t syncStart(const SSyncInfo *pInfo) { ...@@ -182,7 +182,6 @@ int64_t syncStart(const SSyncInfo *pInfo) {
pNode->startSyncFileFp = pInfo->startSyncFileFp; pNode->startSyncFileFp = pInfo->startSyncFileFp;
pNode->stopSyncFileFp = pInfo->stopSyncFileFp; pNode->stopSyncFileFp = pInfo->stopSyncFileFp;
pNode->getVersionFp = pInfo->getVersionFp; pNode->getVersionFp = pInfo->getVersionFp;
pNode->resetVersionFp = pInfo->resetVersionFp;
pNode->sendFileFp = pInfo->sendFileFp; pNode->sendFileFp = pInfo->sendFileFp;
pNode->recvFileFp = pInfo->recvFileFp; pNode->recvFileFp = pInfo->recvFileFp;
......
...@@ -238,7 +238,6 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { ...@@ -238,7 +238,6 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
(*pNode->stopSyncFileFp)(pNode->vgId, fversion); (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
nodeVersion = fversion; nodeVersion = fversion;
if (pNode->resetVersionFp) (*pNode->resetVersionFp)(pNode->vgId, fversion);
sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
uint64_t wver = 0; uint64_t wver = 0;
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODE_BACKUP_H
#define TDENGINE_VNODE_BACKUP_H
#ifdef __cplusplus
extern "C" {
#endif
#include "vnodeInt.h"
int32_t vnodeInitBackup();
void vnodeCleanupBackup();
int32_t vnodeBackup(int32_t vgId);
#ifdef __cplusplus
}
#endif
#endif
...@@ -30,7 +30,6 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); ...@@ -30,7 +30,6 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion);
void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);
int32_t vnodeResetVersion(int32_t vgId, uint64_t fver);
void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force);
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tutil.h"
#include "tqueue.h"
#include "tglobal.h"
#include "tfs.h"
#include "vnodeBackup.h"
#include "vnodeMain.h"
typedef struct {
int32_t vgId;
} SVBackupMsg;
typedef struct {
pthread_t thread;
int32_t workerId;
} SVBackupWorker;
typedef struct {
int32_t num;
SVBackupWorker *worker;
} SVBackupWorkerPool;
static SVBackupWorkerPool tsVBackupPool;
static taos_qset tsVBackupQset;
static taos_queue tsVBackupQueue;
static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) {
int32_t vgId = pMsg->vgId;
char newDir[TSDB_FILENAME_LEN] = {0};
char stagingDir[TSDB_FILENAME_LEN] = {0};
sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
if (tsEnableVnodeBak) {
tfsRmdir(newDir);
tfsRename(stagingDir, newDir);
} else {
vInfo("vgId:%d, vnode backup not enabled", vgId);
tfsRmdir(stagingDir);
}
}
static void *vnodeBackupFunc(void *param) {
while (1) {
SVBackupMsg *pMsg = NULL;
if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) {
vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset);
break;
}
vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId);
vnodeProcessBackupMsg(pMsg);
vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId);
taosFreeQitem(pMsg);
}
return NULL;
}
static int32_t vnodeStartBackup() {
tsVBackupQueue = taosOpenQueue();
if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL);
for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
SVBackupWorker *pWorker = tsVBackupPool.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) {
vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num);
}
vDebug("vbackup queue:%p is allocated", tsVBackupQueue);
return TSDB_CODE_SUCCESS;
}
static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) {
SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg));
if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
pMsg->vgId = vgId;
int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg);
if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS;
return code;
}
int32_t vnodeBackup(int32_t vgId) {
vTrace("vgId:%d, will backup", vgId);
return vnodeWriteIntoBackupWorker(vgId);
}
int32_t vnodeInitBackup() {
tsVBackupQset = taosOpenQset();
tsVBackupPool.num = 1;
tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num);
if (tsVBackupPool.worker == NULL) return -1;
for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
SVBackupWorker *pWorker = tsVBackupPool.worker + i;
pWorker->workerId = i;
vDebug("vbackup:%d is created", i);
}
vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset);
return vnodeStartBackup();
}
void vnodeCleanupBackup() {
for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
SVBackupWorker *pWorker = tsVBackupPool.worker + i;
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsVBackupQset);
}
vDebug("vbackup:%d is closed", i);
}
for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
SVBackupWorker *pWorker = tsVBackupPool.worker + i;
vDebug("vbackup:%d start to join", i);
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
vDebug("vbackup:%d join success", i);
}
vDebug("vbackup is closed, qset:%p", tsVBackupQset);
taosCloseQset(tsVBackupQset);
tsVBackupQset = NULL;
tfree(tsVBackupPool.worker);
vDebug("vbackup queue:%p is freed", tsVBackupQueue);
taosCloseQueue(tsVBackupQueue);
tsVBackupQueue = NULL;
}
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
#include "vnodeVersion.h" #include "vnodeVersion.h"
#include "vnodeMgmt.h" #include "vnodeMgmt.h"
#include "vnodeWorker.h" #include "vnodeWorker.h"
#include "vnodeBackup.h"
#include "vnodeMain.h" #include "vnodeMain.h"
static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
...@@ -364,7 +365,6 @@ int32_t vnodeOpen(int32_t vgId) { ...@@ -364,7 +365,6 @@ int32_t vnodeOpen(int32_t vgId) {
syncInfo.startSyncFileFp = vnodeStartSyncFile; syncInfo.startSyncFileFp = vnodeStartSyncFile;
syncInfo.stopSyncFileFp = vnodeStopSyncFile; syncInfo.stopSyncFileFp = vnodeStopSyncFile;
syncInfo.getVersionFp = vnodeGetVersion; syncInfo.getVersionFp = vnodeGetVersion;
syncInfo.resetVersionFp = vnodeResetVersion;
syncInfo.sendFileFp = tsdbSyncSend; syncInfo.sendFileFp = tsdbSyncSend;
syncInfo.recvFileFp = tsdbSyncRecv; syncInfo.recvFileFp = tsdbSyncRecv;
syncInfo.pTsdb = pVnode->tsdb; syncInfo.pTsdb = pVnode->tsdb;
...@@ -448,18 +448,14 @@ void vnodeDestroy(SVnodeObj *pVnode) { ...@@ -448,18 +448,14 @@ void vnodeDestroy(SVnodeObj *pVnode) {
if (pVnode->dropped) { if (pVnode->dropped) {
char rootDir[TSDB_FILENAME_LEN] = {0}; char rootDir[TSDB_FILENAME_LEN] = {0};
char newDir[TSDB_FILENAME_LEN] = {0}; char stagingDir[TSDB_FILENAME_LEN] = {0};
sprintf(rootDir, "%s/vnode%d", "vnode", vgId); sprintf(rootDir, "%s/vnode%d", "vnode", vgId);
sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
if (0 == tsEnableVnodeBak) { tfsRename(rootDir, stagingDir);
vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
} else { vnodeBackup(vgId);
tfsRmdir(newDir);
tfsRename(rootDir, newDir);
}
tfsRmdir(rootDir);
dnodeSendStatusMsgToMnode(); dnodeSendStatusMsgToMnode();
} }
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
#include "os.h" #include "os.h"
#include "dnode.h" #include "dnode.h"
#include "vnodeStatus.h" #include "vnodeStatus.h"
#include "vnodeBackup.h"
#include "vnodeWorker.h" #include "vnodeWorker.h"
#include "vnodeRead.h" #include "vnodeRead.h"
#include "vnodeWrite.h" #include "vnodeWrite.h"
...@@ -29,6 +30,7 @@ static void vnodeCleanupHash(void); ...@@ -29,6 +30,7 @@ static void vnodeCleanupHash(void);
static void vnodeIncRef(void *ptNode); static void vnodeIncRef(void *ptNode);
static SStep tsVnodeSteps[] = { static SStep tsVnodeSteps[] = {
{"vnode-backup", vnodeInitBackup, vnodeCleanupBackup},
{"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker},
{"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, {"vnode-write", vnodeInitWrite, vnodeCleanupWrite},
{"vnode-read", vnodeInitRead, vnodeCleanupRead}, {"vnode-read", vnodeInitRead, vnodeCleanupRead},
......
...@@ -107,8 +107,9 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { ...@@ -107,8 +107,9 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
pVnode->fversion = fversion; pVnode->fversion = fversion;
pVnode->version = fversion; pVnode->version = fversion;
vnodeSaveVersion(pVnode); vnodeSaveVersion(pVnode);
walResetVersion(pVnode->wal, fversion);
vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
vnodeSetReadyStatus(pVnode); vnodeSetReadyStatus(pVnode);
vnodeRelease(pVnode); vnodeRelease(pVnode);
...@@ -158,22 +159,6 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { ...@@ -158,22 +159,6 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) {
return code; return code;
} }
int32_t vnodeResetVersion(int32_t vgId, uint64_t fver) {
SVnodeObj *pVnode = vnodeAcquire(vgId);
if (pVnode == NULL) {
vError("vgId:%d, vnode not found while reset version", vgId);
return -1;
}
pVnode->fversion = fver;
pVnode->version = fver;
walResetVersion(pVnode->wal, fver);
vInfo("vgId:%d, version reset to %" PRIu64, vgId, fver);
vnodeRelease(pVnode);
return 0;
}
void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) { void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) {
SVnodeObj *pVnode = vparam; SVnodeObj *pVnode = vparam;
syncConfirmForward(pVnode->sync, version, code, force); syncConfirmForward(pVnode->sync, version, code, force);
......
...@@ -96,6 +96,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara ...@@ -96,6 +96,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
// write into WAL // write into WAL
code = walWrite(pVnode->wal, pHead); code = walWrite(pVnode->wal, pHead);
if (code < 0) { if (code < 0) {
if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code);
return code; return code;
} }
...@@ -104,7 +105,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara ...@@ -104,7 +105,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
// write data locally // write data locally
code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet);
if (code < 0) return code; if (code < 0) {
if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
return code;
}
return syncCode; return syncCode;
} }
......
...@@ -68,12 +68,12 @@ public class JDBCDemo { ...@@ -68,12 +68,12 @@ public class JDBCDemo {
} }
private void insert() { private void insert() {
final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)";
exuete(sql); exuete(sql);
} }
private void select() { private void select() {
final String sql = "select * from test.weather"; final String sql = "select * from "+ dbName + "." + tbName;
executeQuery(sql); executeQuery(sql);
} }
......
...@@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py ...@@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py
python3 ./test.py -f import_merge/importCSV.py python3 ./test.py -f import_merge/importCSV.py
#======================p1-end=============== #======================p1-end===============
#======================p2-start=============== #======================p2-start===============
# tools
python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdemoTestWithoutMetric.py
python3 test.py -f tools/taosdemoTestWithJson.py
python3 test.py -f tools/taosdemoTestLimitOffset.py
python3 test.py -f tools/taosdemoTestTblAlt.py
python3 test.py -f tools/taosdemoTestSampleData.py
python3 test.py -f tools/taosdemoTestInterlace.py
python3 test.py -f tools/taosdemoTestQuery.py
# update # update
python3 ./test.py -f update/allow_update.py python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py python3 ./test.py -f update/allow_update-0.py
...@@ -201,6 +213,8 @@ python3 ./test.py -f query/bug2119.py ...@@ -201,6 +213,8 @@ python3 ./test.py -f query/bug2119.py
python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/isNullTest.py
python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/queryWithTaosdKilled.py
python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/floatCompare.py
python3 ./test.py -f query/query1970YearsAf.py
python3 ./test.py -f query/bug3351.py
python3 ./test.py -f query/bug3375.py python3 ./test.py -f query/bug3375.py
...@@ -247,18 +261,6 @@ python3 test.py -f subscribe/supertable.py ...@@ -247,18 +261,6 @@ python3 test.py -f subscribe/supertable.py
#======================p3-end=============== #======================p3-end===============
#======================p4-start=============== #======================p4-start===============
# tools
python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdemoTestWithoutMetric.py
python3 test.py -f tools/taosdemoTestWithJson.py
python3 test.py -f tools/taosdemoTestLimitOffset.py
python3 test.py -f tools/taosdemoTest2.py
python3 test.py -f tools/taosdemoTestSampleData.py
python3 test.py -f tools/taosdemoTestInterlace.py
python3 test.py -f tools/taosdemoTestQuery.py
python3 ./test.py -f update/merge_commit_data-0.py python3 ./test.py -f update/merge_commit_data-0.py
# wal # wal
python3 ./test.py -f wal/addOldWalTest.py python3 ./test.py -f wal/addOldWalTest.py
......
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
tdSql.execute("drop database if exists db")
tdSql.execute("create database if not exists db keep 36500")
tdSql.execute("use db")
tdLog.printNoPrefix("==========step1:create table && insert data")
tdSql.execute(
"create table stb1 (ts timestamp, c1 int) TAGS(t1 int)"
)
tdSql.execute("create table t0 using stb1 tags(1)")
tdSql.execute("insert into t0 values (-865000000, 1)")
tdSql.execute("insert into t0 values (-864000000, 2)")
tdSql.execute("insert into t0 values (-863000000, 3)")
tdSql.execute("insert into t0 values (-15230000, 4)")
tdSql.execute("insert into t0 values (-15220000, 5)")
tdSql.execute("insert into t0 values (-15210000, 6)")
tdLog.printNoPrefix("==========step2:query")
# bug1:when ts > -864000000, return 0 rows;
# bug2:when ts = -15220000, return 0 rows.
tdSql.query('select * from t0 where ts < -864000000')
tdSql.checkRows(1)
tdSql.query('select * from t0 where ts <= -864000000')
tdSql.checkRows(2)
tdSql.query('select * from t0 where ts = -864000000')
tdSql.checkRows(1)
tdSql.query('select * from t0 where ts > -864000000')
tdSql.checkRows(4)
tdSql.query('select * from t0 where ts >= -864000000')
tdSql.checkRows(5)
tdSql.query('select * from t0 where ts < -15220000')
tdSql.checkRows(4)
tdSql.query('select * from t0 where ts <= -15220000')
tdSql.checkRows(5)
tdSql.query('select * from t0 where ts = -15220000')
tdSql.checkRows(1)
tdSql.query('select * from t0 where ts > -15220000')
tdSql.checkRows(1)
tdSql.query('select * from t0 where ts >= -15220000')
tdSql.checkRows(2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
...@@ -22,25 +22,34 @@ import datetime ...@@ -22,25 +22,34 @@ import datetime
from util.log import * from util.log import *
from util.sql import * from util.sql import *
from util.cases import * from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnode
class TDTestCase: class TDTestCase:
def __init__(self):
self.path = ""
def init(self, conn, logSql): def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor()) tdSql.init(conn.cursor())
def getCfgDir(self, path): def getcfgPath(self, path):
binPath = os.path.dirname(os.path.realpath(__file__)) binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/" binPath = binPath + "/../../../debug/"
tdLog.debug("binPath %s" % (binPath)) tdLog.debug(f"binPath {binPath}")
binPath = os.path.realpath(binPath) binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath)) tdLog.debug(f"binPath real path {binPath}")
if path == "": if path == "":
self.path = os.path.abspath(binPath + "../../") self.path = os.path.abspath(binPath + "../../")
else: else:
self.path = os.path.realpath(path) self.path = os.path.realpath(path)
return self.path
self.cfgDir = "%s/sim/psim/cfg" % (self.path) def getCfgDir(self):
self.getcfgPath(self.path)
self.cfgDir = f"{self.path}/sim/psim/cfg"
return self.cfgDir return self.cfgDir
def creatcfg(self): def creatcfg(self):
...@@ -63,7 +72,7 @@ class TDTestCase: ...@@ -63,7 +72,7 @@ class TDTestCase:
"update": 0 "update": 0
} }
# 设置创建的超级表格式 # set stable schema
stable1 = { stable1 = {
"name": "stb2", "name": "stb2",
"child_table_exists": "no", "child_table_exists": "no",
...@@ -75,35 +84,49 @@ class TDTestCase: ...@@ -75,35 +84,49 @@ class TDTestCase:
"insert_rows": 5000, "insert_rows": 5000,
"multi_thread_write_one_tbl": "no", "multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 0, "number_of_tbl_in_one_sql": 0,
"rows_per_tbl": 1000, "rows_per_tbl": 1,
"max_sql_len": 65480, "max_sql_len": 65480,
"disorder_ratio": 0, "disorder_ratio": 0,
"disorder_range": 1000, "disorder_range": 1000,
"timestamp_step": 19000000, "timestamp_step": 20000,
"start_timestamp": "1969-01-01 00:00:00.000", "start_timestamp": "1969-12-31 00:00:00.000",
"sample_format": "csv", "sample_format": "csv",
"sample_file": "./sample.csv", "sample_file": "./sample.csv",
"tags_file": "", "tags_file": "",
"columns": [ "columns": [
{"type": "INT"}, {"type": "INT", "count": 2},
{"type": "DOUBLE", "count": 10}, {"type": "DOUBLE", "count": 2},
{"type": "BINARY", "len": 16, "count": 3}, {"type": "BIGINT", "count": 2},
{"type": "BINARY", "len": 32, "count": 6} {"type": "FLOAT", "count": 2},
{"type": "SMALLINT", "count": 2},
{"type": "TINYINT", "count": 2},
{"type": "BOOL", "count": 2},
{"type": "NCHAR", "len": 3, "count": 1},
{"type": "BINARY", "len": 8, "count": 1}
], ],
"tags": [ "tags": [
{"type": "INT", "count": 2},
{"type": "DOUBLE", "count": 2},
{"type": "BIGINT", "count": 2},
{"type": "FLOAT", "count": 2},
{"type": "SMALLINT", "count": 2},
{"type": "TINYINT", "count": 2}, {"type": "TINYINT", "count": 2},
{"type": "BINARY", "len": 16, "count": 5} {"type": "BOOL", "count": 2},
{"type": "NCHAR", "len": 3, "count": 1},
{"type": "BINARY", "len": 8, "count": 1}
] ]
} }
# 需要创建多个超级表时,只需创建不同的超级表格式并添加至super_tables # create different stables like stable1 and add to list super_tables
super_tables = [stable1] super_tables = []
super_tables.append(stable1)
database = { database = {
"dbinfo": dbinfo, "dbinfo": dbinfo,
"super_tables": super_tables "super_tables": super_tables
} }
cfgdir = self.getCfgDir("") cfgdir = self.getCfgDir()
create_table = { create_table = {
"filetype": "insert", "filetype": "insert",
"cfgdir": cfgdir, "cfgdir": cfgdir,
...@@ -121,105 +144,110 @@ class TDTestCase: ...@@ -121,105 +144,110 @@ class TDTestCase:
} }
return create_table return create_table
def inserttable(self): def createinsertfile(self):
create_table = self.creatcfg() create_table = self.creatcfg()
date = datetime.datetime.now().strftime("%Y%m%d%H%M") date = datetime.datetime.now().strftime("%Y%m%d%H%M")
file_create_table = f"/tmp/insert_{date}.json" file_create_table = f"/tmp/insert_{date}.json"
with open(file_create_table, 'w') as f: with open(file_create_table, 'w') as f:
json.dump(create_table, f) json.dump(create_table, f)
return file_create_table
create_table_cmd = f"taosdemo -f {file_create_table}" def inserttable(self, filepath):
create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1"
_ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8")
def sqlsquery(self):
def run(self):
s = 'reset query cache'
tdSql.execute(s)
s = 'create database if not exists db'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
tdLog.info("==========step1:create table stable and child table,then insert data automatically")
self.inserttable()
# tdSql.execute(
# '''create table if not exists supt
# (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
# tags(location binary(64), type int, isused bool , family nchar(64))'''
# )
# tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')")
# tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')")
# tdSql.execute(
# f"insert into t1 values (-31564800000, 6, 5, 4, 3, 2, 1)"
# )
tdLog.info("==========step2:query join")
# stable query # stable query
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(16600) tdSql.checkRows(43200)
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(33400) tdSql.checkRows(6800)
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts > '1969-12-01 00:00:00.000' and stb2.ts <'1970-01-31 00:00:00.000' " "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' "
) )
tdSql.checkRows(2780) tdSql.checkRows(3590)
# child-table query # child-tables query
tdSql.query( tdSql.query(
"select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(1660) tdSql.checkRows(4320)
tdSql.query( tdSql.query(
"select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(3340) tdSql.checkRows(680)
tdSql.query( tdSql.query(
"select * from t9 where t9.ts > '1969-12-01 00:00:00.000' and t9.ts <'1970-01-31 00:00:00.000' " "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' "
) )
tdSql.checkRows(278) tdSql.checkRows(719)
tdSql.query( tdSql.query(
"select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(3340) tdSql.checkRows(680)
tdSql.query( tdSql.query(
"select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
) )
tdSql.checkRows(3339) tdSql.checkRows(679)
tdSql.query( tdSql.query(
"select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts"
) )
tdSql.checkRows(16600) tdSql.checkRows(43200)
# query with timestamp in 'where ...' # query with timestamp in 'where ...'
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts > -28800000 " "select * from stb2 where stb2.ts > -28800000 "
) )
tdSql.checkRows(33400) tdSql.checkRows(6790)
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' "
) )
tdSql.checkRows(20) tdSql.checkRows(6790)
tdSql.query( tdSql.query(
"select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 16:00:00.000' " "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' "
) )
tdSql.checkRows(3590)
def run(self):
s = 'reset query cache'
tdSql.execute(s)
s = 'create database if not exists db'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
tdLog.info("==========step1:create table stable and child table,then insert data automatically")
insertfile = self.createinsertfile()
self.inserttable(insertfile)
tdLog.info("==========step2:query join")
self.sqlsquery()
# after wal and sync, check again
tdSql.query("show dnodes")
index = tdSql.getData(0, 0)
tdDnodes.stop(index)
tdDnodes.start(index)
tdLog.info("==========step3: query join again")
self.sqlsquery()
# delete temporary file
rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1"
_ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8")
def stop(self): def stop(self):
tdSql.close() tdSql.close()
......
...@@ -29,10 +29,33 @@ class TDTestCase: ...@@ -29,10 +29,33 @@ class TDTestCase:
self.numberOfTables = 10 self.numberOfTables = 10
self.numberOfRecords = 1000000 self.numberOfRecords = 1000000
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def insertDataAndAlterTable(self, threadID): def insertDataAndAlterTable(self, threadID):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
if(threadID == 0): if(threadID == 0):
os.system("taosdemo -y -t %d -n %d" % os.system("%staosdemo -y -t %d -n %d" %
(self.numberOfTables, self.numberOfRecords)) (binPath, self.numberOfTables, self.numberOfRecords))
if(threadID == 1): if(threadID == 1):
time.sleep(2) time.sleep(2)
print("use test") print("use test")
...@@ -47,7 +70,13 @@ class TDTestCase: ...@@ -47,7 +70,13 @@ class TDTestCase:
# check if all the tables have heen created # check if all the tables have heen created
while True: while True:
tdSql.query("show tables") try:
tdSql.query("show tables")
except Exception as e:
tdLog.info("show tables test failed")
time.sleep(1)
continue
rows = tdSql.queryRows rows = tdSql.queryRows
print("number of tables: %d" % rows) print("number of tables: %d" % rows)
if(rows == self.numberOfTables): if(rows == self.numberOfTables):
...@@ -56,16 +85,23 @@ class TDTestCase: ...@@ -56,16 +85,23 @@ class TDTestCase:
# check if there are any records in the last created table # check if there are any records in the last created table
while True: while True:
print("query started") print("query started")
tdSql.query("select * from test.t9") try:
tdSql.query("select * from test.t9")
except Exception as e:
tdLog.info("select * test failed")
time.sleep(2)
continue
rows = tdSql.queryRows rows = tdSql.queryRows
print("number of records: %d" % rows) print("number of records: %d" % rows)
if(rows > 0): if(rows > 0):
break break
time.sleep(1) time.sleep(1)
print("alter table test.meters add column col10 int") print("alter table test.meters add column col10 int")
tdSql.execute("alter table test.meters add column col10 int") tdSql.execute("alter table test.meters add column col10 int")
print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
def run(self): def run(self):
tdSql.prepare() tdSql.prepare()
...@@ -78,6 +114,8 @@ class TDTestCase: ...@@ -78,6 +114,8 @@ class TDTestCase:
t1.join() t1.join()
t2.join() t2.join()
time.sleep(3)
tdSql.query("select count(*) from test.meters") tdSql.query("select count(*) from test.meters")
tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1)
......
...@@ -25,29 +25,24 @@ function stopTaosd { ...@@ -25,29 +25,24 @@ function stopTaosd {
function dohavecore(){ function dohavecore(){
corefile=`find $corepath -mmin 1` corefile=`find $corepath -mmin 1`
core_file=`echo $corefile|cut -d " " -f2` core_file=`echo $corefile|cut -d " " -f2`
echo "corefile:$core_file"
echo "corepath:$corepath"
ls -l $corepath
proc=`echo $corefile|cut -d "_" -f3` proc=`echo $corefile|cut -d "_" -f3`
if [ -n "$corefile" ];then if [ -n "$corefile" ];then
echo 'taosd or taos has generated core' echo 'taosd or taos has generated core'
rm case.log
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then
cd ../../../ cd ../../../
tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so*
if [[ $2 == 1 ]];then if [[ $2 == 1 ]];then
cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
rm -rf sim/case.log
else else
cd community cd community
cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
rm -rf sim/case.log
fi fi
else else
cd ../../ cd ../../
if [[ $1 == 1 ]];then if [[ $1 == 1 ]];then
tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so*
cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
rm -rf sim/case.log
fi fi
fi fi
if [[ $1 == 1 ]];then if [[ $1 == 1 ]];then
...@@ -82,7 +77,6 @@ function runSimCaseOneByOne { ...@@ -82,7 +77,6 @@ function runSimCaseOneByOne {
# fi # fi
end_time=`date +%s` end_time=`date +%s`
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
dohavecore 0
fi fi
done < $1 done < $1
} }
...@@ -99,26 +93,25 @@ function runSimCaseOneByOnefq { ...@@ -99,26 +93,25 @@ function runSimCaseOneByOnefq {
date +%F\ %T | tee -a out.log date +%F\ %T | tee -a out.log
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
echo -n $case echo -n $case
./test.sh -f $case > ../../../sim/case.log 2>&1 && \ ./test.sh -f $case > case.log 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../../sim/case.log ) ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
else else
echo -n $case echo -n $case
./test.sh -f $case > ../../sim/case.log 2>&1 && \ ./test.sh -f $case > ../../sim/case.log 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../sim/case.log ) ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
fi fi
out_log=`tail -1 out.log ` out_log=`tail -1 out.log `
if [[ $out_log =~ 'failed' ]];then if [[ $out_log =~ 'failed' ]];then
rm case.log
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
rm -rf ../../../sim/case.log
else else
cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
rm -rf ../../sim/case.log
fi fi
dohavecore $2 1 dohavecore $2 1
if [[ $2 == 1 ]];then if [[ $2 == 1 ]];then
...@@ -159,7 +152,6 @@ function runPyCaseOneByOne { ...@@ -159,7 +152,6 @@ function runPyCaseOneByOne {
else else
$line > /dev/null 2>&1 $line > /dev/null 2>&1
fi fi
dohavecore 0
fi fi
done < $1 done < $1
} }
...@@ -185,7 +177,7 @@ function runPyCaseOneByOnefq() { ...@@ -185,7 +177,7 @@ function runPyCaseOneByOnefq() {
start_time=`date +%s` start_time=`date +%s`
date +%F\ %T | tee -a pytest-out.log date +%F\ %T | tee -a pytest-out.log
echo -n $case echo -n $case
$line > ../../sim/case.log 2>&1 && \ $line > case.log 2>&1 && \
echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \
echo -e "${RED} failed${NC}" | tee -a pytest-out.log echo -e "${RED} failed${NC}" | tee -a pytest-out.log
end_time=`date +%s` end_time=`date +%s`
...@@ -193,8 +185,8 @@ function runPyCaseOneByOnefq() { ...@@ -193,8 +185,8 @@ function runPyCaseOneByOnefq() {
if [[ $out_log =~ 'failed' ]];then if [[ $out_log =~ 'failed' ]];then
cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
echo '=====================log===================== ' echo '=====================log===================== '
cat ../../sim/case.log cat case.log
rm -rf ../../sim/case.log rm -rf case.log
dohavecore $2 2 dohavecore $2 2
if [[ $2 == 1 ]];then if [[ $2 == 1 ]];then
exit 8 exit 8
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册