diff --git a/Jenkinsfile b/Jenkinsfile index 536cbe73a29510bcab57e7f906c024ceba37076a..3119b5031934307452a1b502d8606c32194f4fa2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ def pre_test(){ git pull git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0 + git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|.*src/connector|Jenkinsfile' || exit 0 cd ${WK} git reset --hard HEAD~10 git checkout develop diff --git a/deps/MsvcLibX/include/msvcUnistd.h b/deps/MsvcLibX/include/msvcUnistd.h index 9ad60625e07be8e9b749d1951375a3a501832366..9b59bae7f026d1daff65641f665f1df4ca6eb3b5 100644 --- a/deps/MsvcLibX/include/msvcUnistd.h +++ b/deps/MsvcLibX/include/msvcUnistd.h @@ -89,11 +89,12 @@ pid_t getppid(void); /* Get parent PID */ /* Path management */ #if defined(_WIN32) -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) #define realpath realpathU +#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) +// #define realpath realpathU #define CompactPath CompactPathU #else /* _ANSI_SOURCE */ -#define realpath realpathA +// #define realpath realpathA #define CompactPath CompactPathA #endif #endif /* defined(_WIN32) */ diff --git a/deps/rmonotonic/src/monotonic.c b/deps/rmonotonic/src/monotonic.c index 622ac1b075a66ec49338cda3cc4afbc8355636dd..1470f91b56c79b4ee2d8429ecf58fc365d03e737 100644 --- a/deps/rmonotonic/src/monotonic.c +++ b/deps/rmonotonic/src/monotonic.c @@ -84,11 +84,11 @@ static void monotonicInit_x86linux() { regfree(&constTscRegex); if (mono_ticksPerMicrosecond == 0) { - fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate"); + //fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate"); return; } if (!constantTsc) { - fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present"); + //fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present"); return; } diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index bcaabe3c0a48837cc50d1d2f5f212a57c8e5c912..6736eea7c75ec12f017aa272385d4301c7875317 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -852,7 +852,7 @@ npm install td2.0-connector ### Linux - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) -- `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。 +- `node` 2.0.6支持v12.x和v10.x，2.0.5及更早版本支持v10.x版本，其他版本可能存在包兼容性的问题。 - `make` - c语言编译器比如GCC diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 890900caa2fdc50af0d3601030438ad8a426dfd2..95cf28ec49346e500d00c1bcac9dc6f4055a0eb4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1433,9 +1433,9 @@ int32_t tscValidateName(SStrToken* pToken) { if (sep == NULL) { // single part if (pToken->type == TK_STRING) { - strdequote(pToken->z); + tscDequoteAndTrimToken(pToken); tscStrToLower(pToken->z, pToken->n); - pToken->n = (uint32_t)strtrim(pToken->z); + //pToken->n = (uint32_t)strtrim(pToken->z); int len = tSQLGetToken(pToken->z, &pToken->type); diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 980524be965c94bd48185494d864d5dd5701f20e..3d88cf83127672f8f5780da695e24cee4850c035 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -59,7 +59,6 @@ char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -182,6 +181,7 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; +char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; int32_t tsDiskCfgNum = 0; diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 4756ac555fa3b32f1e261b3376d811f5e24bcc70..0626bcf1fb1161cc38ad05002b984b2200b68d68 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -36,6 +36,7 @@ 3.6.0 1.1.2 3.5 + @@ -122,11 +123,14 @@ maven-surefire-plugin 2.12.4 + pertest + ${maven.test.jvmargs} **/*Test.java **/AppMemoryLeakTest.java + **/TaosInfoMonitorTest.java **/FailOverTest.java true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 94abe3965507170a4b31e17ebb431ddcb4fa11f8..547fe6a9e9900981551b138c204d0f24d6ef152b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -34,44 +34,37 @@ import java.util.*; import java.util.concurrent.Executor; public class TSDBConnection implements Connection { - protected Properties props = null; private TSDBJNIConnector connector = null; private String catalog = null; - private TSDBDatabaseMetaData dbMetaData = null; + private TSDBDatabaseMetaData dbMetaData; private Properties clientInfoProps = new Properties(); private int timeoutMilliseconds = 0; - + private boolean batchFetch = false; public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { this.dbMetaData = meta; connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), - info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), + info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); - + String batchLoad = info.getProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD); if (batchLoad != null) { - this.batchFetch = Boolean.parseBoolean(batchLoad); + this.batchFetch = Boolean.parseBoolean(batchLoad); } } private void connect(String host, int port, String dbName, String user, String password) throws SQLException { this.connector = new TSDBJNIConnector(); this.connector.connect(host, port, dbName, user, password); - - try { - this.setCatalog(dbName); - } catch (SQLException e) { - e.printStackTrace(); - } - + this.setCatalog(dbName); this.dbMetaData.setConnection(this); } @@ -80,68 +73,86 @@ public class TSDBConnection implements Connection { } public Statement createStatement() throws SQLException { - if (!this.connector.isClosed()) { - TSDBStatement statement = new TSDBStatement(this, this.connector); - statement.setConnection(this); - return statement; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + + TSDBStatement statement = new TSDBStatement(this, this.connector); + statement.setConnection(this); + return statement; } public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { - if (this.connector.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } long id = this.connector.subscribe(topic, sql, restart, 0); if (id == 0) { - throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED); } - return new TSDBSubscribe(this.connector, id); } public PreparedStatement prepareStatement(String sql) throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBPreparedStatement(this, this.connector, sql); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + + return new TSDBPreparedStatement(this, this.connector, sql); } public CallableStatement prepareCall(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String nativeSQL(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void setAutoCommit(boolean autoCommit) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + } public boolean getAutoCommit() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + return true; } public void commit() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } } public void rollback() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void close() throws SQLException { - if (this.connector != null && !this.connector.isClosed()) { - this.connector.closeConnection(); - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + this.connector.closeConnection(); } public boolean isClosed() throws SQLException { - return this.connector.isClosed(); + return this.connector != null && this.connector.isClosed(); } /** @@ -154,6 +165,9 @@ public class TSDBConnection implements Connection { * @throws SQLException if a database access error occurs */ public DatabaseMetaData getMetaData() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return this.dbMetaData; } @@ -165,17 +179,29 @@ public class TSDBConnection implements Connection { * @throws SQLException */ public void setReadOnly(boolean readOnly) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } } public boolean isReadOnly() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return true; } public void setCatalog(String catalog) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } this.catalog = catalog; } public String getCatalog() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return this.catalog; } @@ -187,6 +213,19 @@ public class TSDBConnection implements Connection { * @throws SQLException */ public void setTransactionIsolation(int level) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + switch (level) { + case Connection.TRANSACTION_NONE: + case Connection.TRANSACTION_READ_COMMITTED: + case Connection.TRANSACTION_READ_UNCOMMITTED: + case Connection.TRANSACTION_REPEATABLE_READ: + case Connection.TRANSACTION_SERIALIZABLE: + break; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } } /** @@ -196,60 +235,81 @@ public class TSDBConnection implements Connection { * @throws SQLException */ public int getTransactionIsolation() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return Connection.TRANSACTION_NONE; } public SQLWarning getWarnings() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } //todo: implement getWarnings according to the warning messages returned from TDengine return null; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public void clearWarnings() throws SQLException { - // left blank to support HikariCP connection + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } //todo: implement clearWarnings according to the warning messages returned from TDengine } public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { // This method is implemented in the current way to support Spark if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } return this.prepareStatement(sql); } - + public Boolean getBatchFetch() { - return this.batchFetch; + return this.batchFetch; } - + public void setBatchFetch(Boolean batchFetch) { - this.batchFetch = batchFetch; + this.batchFetch = batchFetch; } public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Map> getTypeMap() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void setTypeMap(Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void setHoldability(int holdability) throws SQLException { - // intentionally left empty to support druid connection pool. + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } } /** @@ -259,67 +319,111 @@ public class TSDBConnection implements Connection { * @throws SQLException */ public int getHoldability() throws SQLException { - //intentionally left empty to support HikariCP connection. + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return ResultSet.HOLD_CURSORS_OVER_COMMIT; } public Savepoint setSavepoint() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Savepoint setSavepoint(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void rollback(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void releaseSavepoint(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - return this.prepareStatement(sql, resultSetType, resultSetConcurrency); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Clob createClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Blob createBlob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public NClob createNClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public SQLXML createSQLXML() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public boolean isValid(int timeout) throws SQLException { @@ -338,31 +442,52 @@ public class TSDBConnection implements Connection { } public String getClientInfo(String name) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return clientInfoProps.getProperty(name); } public Properties getClientInfo() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return clientInfoProps; } public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void setSchema(String schema) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public String getSchema() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void abort(Executor executor) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { @@ -370,14 +495,21 @@ public class TSDBConnection implements Connection { } public int getNetworkTimeout() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + } return this.timeoutMilliseconds; } public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return iface.isInstance(this); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 4fb172ceb59040e2f531fd91b83c776cd7d89067..0cf33692b05ab5e19e198463dc420b8a07c637a5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -19,12 +19,12 @@ import java.util.Map; public abstract class TSDBConstants { - public static final String STATEMENT_CLOSED = "Statement already closed."; - public static final String DEFAULT_PORT = "6200"; + public static final String STATEMENT_CLOSED = "statement is closed"; public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!"; public static final String INVALID_VARIABLES = "invalid variables"; - public static final String RESULT_SET_IS_CLOSED = "resultSet is closed."; + public static final String RESULT_SET_IS_CLOSED = "resultSet is closed"; + public static final String DEFAULT_PORT = "6200"; public static Map DATATYPE_MAP = null; public static final long JNI_NULL_POINTER = 0L; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java new file mode 100644 index 0000000000000000000000000000000000000000..ede0b4e4e847e758d6f4e335d4077d39096e7afc --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -0,0 +1,31 @@ +package com.taosdata.jdbc; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +public class TSDBError { + private static Map TSDBErrorMap = new HashMap<>(); + + static { + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD, "this operation is NOT supported currently!"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variables"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED, "statement is closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED, "resultSet is closed"); + /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription"); + } + + public static String wrapErrMsg(String msg) { + return "TDengine Error: " + msg; + } + + public static SQLException createSQLException(int errorNumber) { + // JDBC exception code is less than 0x2350 + if (errorNumber <= 0x2350) + return new SQLException(TSDBErrorMap.get(errorNumber)); + // JNI exception code is + return new SQLException(wrapErrMsg(TSDBErrorMap.get(errorNumber))); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java new file mode 100644 index 0000000000000000000000000000000000000000..74dbb8ab9af0a19a2d969e9fefc0c863f444a77b --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -0,0 +1,15 @@ +package com.taosdata.jdbc; + +public class TSDBErrorNumbers { + + public static final int ERROR_CONNECTION_CLOSED = 0x2301; // connection already closed + public static final int ERROR_UNSUPPORTED_METHOD = 0x2302; //this operation is NOT supported currently! + public static final int ERROR_INVALID_VARIABLE = 0x2303; //invalid variables + public static final int ERROR_STATEMENT_CLOSED = 0x2304; //statement already closed + public static final int ERROR_RESULTSET_CLOSED = 0x2305; //resultSet is closed + + public static final int ERROR_SUBSCRIBE_FAILED = 0x2350; //failed to create subscription + + private TSDBErrorNumbers() { + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index f918463439fba293171827001acf6930ab271b81..349a02fb37c28b263bb73bcc05f644bc53f71079 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -14,6 +14,8 @@ *****************************************************************************/ package com.taosdata.jdbc; +import com.taosdata.jdbc.utils.TaosInfo; + import java.sql.SQLException; import java.sql.SQLWarning; import java.util.List; @@ -21,6 +23,8 @@ import java.util.List; public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; + private TaosInfo taosInfo = TaosInfo.getInstance(); + static { System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); @@ -91,7 +95,8 @@ public class TSDBJNIConnector { */ public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { - this.closeConnectionImp(this.taos); +// this.closeConnectionImp(this.taos); + closeConnection(); this.taos = TSDBConstants.JNI_NULL_POINTER; } @@ -99,7 +104,8 @@ public class TSDBJNIConnector { if (this.taos == TSDBConstants.JNI_NULL_POINTER) { throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l)); } - + // invoke connectImp only here + taosInfo.conn_open_increment(); return true; } @@ -120,6 +126,7 @@ public class TSDBJNIConnector { Long pSql = 0l; try { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); + taosInfo.stmt_count_increment(); } catch (Exception e) { e.printStackTrace(); this.freeResultSetImp(this.taos, pSql); @@ -244,10 +251,11 @@ public class TSDBJNIConnector { private native int fetchRowImp(long connection, long resultSet, TSDBResultSetRowData rowData); public int fetchBlock(long resultSet, TSDBResultSetBlockData blockData) { - return this.fetchBlockImp(this.taos, resultSet, blockData); + return this.fetchBlockImp(this.taos, resultSet, blockData); } - + private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData); + /** * Execute close operation from C to release connection pointer by JNI * @@ -262,6 +270,8 @@ public class TSDBJNIConnector { } else { throw new SQLException("Undefined error code returned by TDengine when closing a connection"); } + // invoke closeConnectionImpl only here + taosInfo.connect_close_increment(); } private native int closeConnectionImp(long connection); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index e7317b8e1d6eb2744c50a5bf1d106c7b687cc965..82a6b4a3fff634b8bedfe338aeede940860e866a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -14,13 +14,14 @@ *****************************************************************************/ package com.taosdata.jdbc; +import com.taosdata.jdbc.utils.TaosInfo; + import java.sql.*; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; public class TSDBStatement implements Statement { - private TSDBJNIConnector connector = null; + private TSDBJNIConnector connector; /** * To store batched commands @@ -67,13 +68,12 @@ public class TSDBStatement implements Statement { } public ResultSet executeQuery(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); } // TODO make sure it is not a update query pSql = this.connector.executeQuery(sql); - long resultSetPointer = this.connector.getResultSet(); if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { this.connector.freeResultSet(pSql); @@ -98,8 +98,8 @@ public class TSDBStatement implements Statement { } public int executeUpdate(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); } // TODO check if current query is update query @@ -131,25 +131,33 @@ public class TSDBStatement implements Statement { } public int getMaxFieldSize() throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + return 0; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public void setMaxFieldSize(int max) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.STATEMENT_CLOSED); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public int getMaxRows() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.STATEMENT_CLOSED); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } // always set maxRows to zero, meaning unlimitted rows in a resultSet return 0; } public void setMaxRows(int max) throws SQLException { + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } // always set maxRows to zero, meaning unlimited rows in a resultSet } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java deleted file mode 100644 index 800265868da6da5047453d542fc9e37c69fe1e36..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ /dev/null @@ -1,272 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class TDNode { - - private int index; - private int running; - private int deployed; - private boolean testCluster; - private String path; - private String cfgDir; - private String dataDir; - private String logDir; - private String cfgPath; - - public TDNode(int index) { - this.index = index; - running = 0; - deployed = 0; - testCluster = false; - } - - public void setPath(String path) { - this.path = path; - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void setRunning(int running) { - this.running = running; - } - - public void searchTaosd(File dir, ArrayList taosdPath) { - File[] fileList = dir.listFiles(); - - if(fileList == null || fileList.length == 0) { - return; - } - - for(File file : fileList) { - if(file.isFile()) { - if(file.getName().equals("taosd")) { - taosdPath.add(file.getAbsolutePath()); - } - } else { - searchTaosd(file, taosdPath); - } - } - } - - public void start() { - String selfPath = System.getProperty("user.dir"); - String binPath = ""; - String projDir = selfPath + "/../../../"; - - try { - ArrayList taosdPath = new ArrayList<>(); - - File dir = new File(projDir); - String realProjDir = dir.getCanonicalPath(); - dir = new File(realProjDir); - System.out.println("project Dir: " + projDir); - searchTaosd(dir, taosdPath); - - if(taosdPath.size() == 0) { - System.out.println("The project path doens't exist"); - return; - } else { - for(String p : taosdPath) { - if(!p.contains("packaging")) { - binPath = p; - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - if(binPath.isEmpty()) { - System.out.println("taosd not found"); - return; - } else { - System.out.println("taosd found in " + binPath); - } - - if(this.deployed == 0) { - System.out.println("dnode" + index + "is not deployed"); - return; - } - - String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; - System.out.println("start taosd cmd: " + cmd); - - try{ - Runtime.getRuntime().exec(cmd); - TimeUnit.SECONDS.sleep(5); - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 1; - } - - public Integer getTaosdPid() { - String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; - String[] cmds = {"sh", "-c", cmd}; - try { - Process process = Runtime.getRuntime().exec(cmds); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line = null; - Integer res = null; - while((line = reader.readLine()) != null) { - if(!line.isEmpty()) { - res = Integer.valueOf(line); - break; - } - } - - return res; - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public void stop() { - - if (this.running != 0) { - Integer pid = null; - while((pid = getTaosdPid()) != null) { - - String killCmd = "kill -term " + pid; - String[] killCmds = {"sh", "-c", killCmd}; - try { - Runtime.getRuntime().exec(killCmds).waitFor(); - - TimeUnit.SECONDS.sleep(2); - } catch (Exception e) { - e.printStackTrace(); - } - } - - try { - for(int port = 6030; port < 6041; port ++) { - String fuserCmd = "fuser -k -n tcp " + port; - Runtime.getRuntime().exec(fuserCmd).waitFor(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 0; - System.out.println("dnode:" + this.index + " is stopped by kill -term"); - } - } - - public void startIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void stopIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void setCfgConfig(String option, String value) { - try{ - String cmd = "echo " + option + " " + value + " >> " + this.cfgPath; - String[] cmdLine = {"sh", "-c", cmd}; - Process ps = Runtime.getRuntime().exec(cmdLine); - ps.waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public String getDnodeRootDir() { - String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index; - return dnodeRootDir; - } - - public String getDnodesRootDir() { - String dnodesRootDir = this.path + "/sim/psim" + this.index; - return dnodesRootDir; - } - - public void deploy() { - this.logDir = this.path + "/sim/dnode" + this.index + "/log"; - this.dataDir = this.path + "/sim/dnode" + this.index + "/data"; - this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg"; - this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg"; - - try { - String cmd = "rm -rf " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "touch " + this.cfgPath; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - - if(this.testCluster) { - startIP(); - setCfgConfig("masterIp", "192.168.0.1"); - setCfgConfig("secondIp", "192.168.0.2"); - setCfgConfig("publicIp", "192.168.0." + this.index); - setCfgConfig("internalIp", "192.168.0." + this.index); - setCfgConfig("privateIp", "192.168.0." + this.index); - } - setCfgConfig("dataDir", this.dataDir); - setCfgConfig("logDir", this.logDir); - setCfgConfig("numOfLogLines", "1000000/00"); - setCfgConfig("mnodeEqualVnodeNum", "0"); - setCfgConfig("walLevel", "1"); - setCfgConfig("statusInterval", "1"); - setCfgConfig("numOfMnodes", "3"); - setCfgConfig("numOfThreadsPerCore", "2.0"); - setCfgConfig("monitor", "0"); - setCfgConfig("maxVnodeConnections", "30000"); - setCfgConfig("maxMgmtConnections", "30000"); - setCfgConfig("maxMeterConnections", "30000"); - setCfgConfig("maxShellConns", "30000"); - setCfgConfig("locale", "en_US.UTF-8"); - setCfgConfig("charset", "UTF-8"); - setCfgConfig("asyncLog", "0"); - setCfgConfig("anyIp", "0"); - setCfgConfig("dDebugFlag", "135"); - setCfgConfig("mDebugFlag", "135"); - setCfgConfig("sdbDebugFlag", "135"); - setCfgConfig("rpcDebugFlag", "135"); - setCfgConfig("tmrDebugFlag", "131"); - setCfgConfig("cDebugFlag", "135"); - setCfgConfig("httpDebugFlag", "135"); - setCfgConfig("monitorDebugFlag", "135"); - setCfgConfig("udebugFlag", "135"); - setCfgConfig("jnidebugFlag", "135"); - setCfgConfig("qdebugFlag", "135"); - this.deployed = 1; - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java deleted file mode 100644 index efc4c53e2864789811ca30110f9b69e5f685a89e..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.File; -import java.util.*; - -public class TDNodes { - private ArrayList tdNodes; - private boolean testCluster; - - public TDNodes () { - tdNodes = new ArrayList<>(); - for(int i = 1; i < 11; i ++) { - tdNodes.add(new TDNode(i)); - } - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void check(int index) { - if(index < 1 || index > 10) { - System.out.println("index: " + index + " should on a scale of [1, 10]"); - return; - } - } - - public void deploy(int index) { - try { - File file = new File(System.getProperty("user.dir") + "/../../../"); - String projectRealPath = file.getCanonicalPath(); - check(index); - tdNodes.get(index - 1).setTestCluster(this.testCluster); - tdNodes.get(index - 1).setPath(projectRealPath); - tdNodes.get(index - 1).deploy(); - } catch (Exception e) { - e.printStackTrace(); - System.out.println("deploy Test Exception"); - } - } - - public void cfg(int index, String option, String value) { - check(index); - tdNodes.get(index - 1).setCfgConfig(option, value); - } - - public TDNode getTDNode(int index) { - check(index); - return tdNodes.get(index - 1); - } - - public void start(int index) { - check(index); - tdNodes.get(index - 1).start(); - } - - public void stop(int index) { - check(index); - tdNodes.get(index - 1).stop(); - } - - public void startIP(int index) { - check(index); - tdNodes.get(index - 1).startIP(); - } - - public void stopIP(int index) { - check(index); - tdNodes.get(index - 1).stopIP(); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..ee1364ce217c639cbf864f87f83ad6f5ddd6c137 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import javax.management.*; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicLong; + +public class TaosInfo implements TaosInfoMBean { + + private static volatile TaosInfo instance; + private AtomicLong connect_open = new AtomicLong(); + private AtomicLong connect_close = new AtomicLong(); + private AtomicLong statement_count = new AtomicLong(); + + static { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo"); + server.registerMBean(TaosInfo.getInstance(), name); + + } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) { + e.printStackTrace(); + } + } + + @Override + public long getConnect_open() { + return connect_open.get(); + } + + @Override + public long getConnect_close() { + return connect_close.get(); + } + + @Override + public long getConnect_active() { + return connect_open.get() - connect_close.get(); + } + + @Override + public long getStatement_count() { + return statement_count.get(); + } + + /*******************************************************/ + + public void conn_open_increment() { + connect_open.incrementAndGet(); + } + + public void connect_close_increment() { + connect_close.incrementAndGet(); + } + + public void stmt_count_increment() { + statement_count.incrementAndGet(); + } + + /********************************************************************************/ + private TaosInfo() { + } + + public static TaosInfo getInstance() { + if (instance == null) { + synchronized (TaosInfo.class) { + if (instance == null) { + instance = new TaosInfo(); + } + } + } + return instance; + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java new file mode 100644 index 0000000000000000000000000000000000000000..e16f41b2f585244721e923a963c53fb6b7bccf47 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java @@ -0,0 +1,13 @@ +package com.taosdata.jdbc.utils; + +public interface TaosInfoMBean { + + long getConnect_open(); + + long getConnect_close(); + + long getConnect_active(); + + long getStatement_count(); + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..e9e36e20c4f83e98d6dd808da3a58ee628f418f0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java @@ -0,0 +1,49 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.*; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class TaosInfoMonitorTest { + + @Test + public void testCreateTooManyConnection() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; + + List connectionList = IntStream.range(0, 100).mapToObj(i -> { + try { + TimeUnit.MILLISECONDS.sleep(100); + return DriverManager.getConnection(url); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + return null; + }).collect(Collectors.toList()); + + connectionList.stream().forEach(conn -> { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show databases"); + while (rs.next()) { + + } + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + + connectionList.stream().forEach(conn -> { + try { + conn.close(); + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + } +} diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 2cad664307d353502d4b3465417fa9b8827859cc..dba234d7a4360f2cd5167261f513ddcc1c31094d 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.4", + version="2.0.5", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index ada83d72c5ed9a493109c8e12f1cd3ba3e4954cf..bc3b6c65d80cb2f025213f57a6eb728a182ff4a0 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading class TDengineCursor(object): @@ -36,7 +35,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index bb191b5ff1c4319c393b3990d7b055230b863aa6..fb0c1508cb051c577fed9bde534d55e297822fe2 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -91,6 +91,20 @@ void cqRmFromList(SCqObj *pObj) { } +static void freeSCqContext(void *handle) { + if (handle == NULL) { + return; + } + SCqContext *pContext = handle; + pthread_mutex_destroy(&pContext->mutex); + + taosTmrCleanUp(pContext->tmrCtrl); + pContext->tmrCtrl = NULL; + cDebug("vgId:%d, CQ is closed", pContext->vgId); + free(pContext); +} + + void cqFree(void *handle) { if (tsEnableStream == 0) { return; @@ -125,13 +139,7 @@ void cqFree(void *handle) { pthread_mutex_unlock(&pContext->mutex); if (delete) { - pthread_mutex_destroy(&pContext->mutex); - - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - - cDebug("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); + freeSCqContext(pContext); } } @@ -184,18 +192,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { return pContext; } -static void freeSCqContext(void *handle) { - if (handle == NULL) { - return; - } - SCqContext *pContext = handle; - pthread_mutex_destroy(&pContext->mutex); - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - cDebug("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); -} void cqClose(void *handle) { if (tsEnableStream == 0) { return; @@ -204,7 +201,9 @@ void cqClose(void *handle) { if (handle == NULL) return; pContext->delete = 1; - + int32_t hasCq = 0; + int32_t existLoop = 0; + // stop all CQs cqStop(pContext); @@ -218,7 +217,13 @@ void cqClose(void *handle) { cqRmFromList(pObj); rid = pObj->rid; - } else { + + hasCq = 1; + + if (pContext->pHead == NULL) { + existLoop = 1; + } + } else { pthread_mutex_unlock(&pContext->mutex); break; } @@ -226,9 +231,15 @@ void cqClose(void *handle) { pthread_mutex_unlock(&pContext->mutex); taosRemoveRef(cqObjRef, rid); + + if (existLoop) { + break; + } } - freeSCqContext(pContext); + if (hasCq == 0) { + freeSCqContext(pContext); + } } void cqStart(void *handle) { diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index a40df626e27dc02c2c8a81e53b3fe04155ea977f..9a226b81e8eddc13aa5cb37ca422b68bcb808e03 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -36,12 +36,12 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; - + // the following message shall be treated as mnode write dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; @@ -57,13 +57,13 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; - - // the following message shall be treated as mnode query + + // the following message shall be treated as mnode query dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; @@ -153,7 +153,7 @@ static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char * } static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; + if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; @@ -164,7 +164,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char rpcMsg.pCont = pMsg; rpcMsg.contLen = sizeof(SAuthMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; - + dDebug("user:%s, send auth msg to mnodes", user); SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); @@ -202,14 +202,14 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); terrno = rpcRsp.code; - + if (rpcRsp.code != 0) { rpcFreeCont(rpcRsp.pCont); dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid); return NULL; } else { dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid); - + // delete this after debug finished SMDCreateTableMsg *pTable = rpcRsp.pCont; int16_t numOfColumns = htons(pTable->numOfColumns); @@ -231,4 +231,4 @@ SStatisInfo dnodeGetStatisInfo() { } return info; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c index 8878299d94e85b0055f9622c969421e485ce47cd..eda6fb227dd6bf29499933383bb82f83ec851ee4 100644 --- a/src/dnode/src/dnodeStep.c +++ b/src/dnode/src/dnodeStep.c @@ -69,6 +69,6 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { return 0; } -void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { +void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { taosStepCleanupImp(pSteps, stepSize - 1); -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 33ebc8b64ff3c0312c4bc0251d15714f702043fc..e49b3eba99408c9453189106dc6f01e0d0afc7fe 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -181,4 +181,4 @@ static void sigintHandler(int32_t signum, void *sigInfo, void *context) { #ifdef WINDOWS tsem_wait(&exitSem); #endif -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 93d1611ebc67530edcc5f0282f0e26f6e53765ef..a3ff459396a974eae26be3561e418b5836b90f4b 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -53,7 +53,7 @@ void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(pWorker->qset); + if (pWorker->qset) taosQsetThreadResume(pWorker->qset); } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 1d77a6bb6395d7b171a3d79ee75df27ec1826f34..4d8c64d8b319072c857ac800fdc5672c3b05214c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -908,6 +908,7 @@ int main(int argc, char *argv[]) { } pthread_join(read_id, NULL); taos_close(rInfo->taos); + free(rInfo); } taos_cleanup(); diff --git a/src/kit/taosdemox/taosdemox.c b/src/kit/taosdemox/taosdemox.c index a6d962df55670d7b188cfe4ffcf2df9a4be3594a..40ca2323275725f0a496a702f9e0a14e221d0f74 100644 --- a/src/kit/taosdemox/taosdemox.c +++ b/src/kit/taosdemox/taosdemox.c @@ -93,6 +93,8 @@ extern char configDir[]; #define MAX_QUERY_SQL_COUNT 10 #define MAX_QUERY_SQL_LENGTH 256 +#define MAX_DATABASE_COUNT 256 + typedef enum CREATE_SUB_TALBE_MOD_EN { PRE_CREATE_SUBTBL, AUTO_CREATE_SUBTBL, @@ -116,7 +118,41 @@ enum QUERY_TYPE { INSERT_TYPE, QUERY_TYPE_BUT } ; - + +enum _show_db_index { + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB +}; + +// -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- +enum _show_stables_index { + TSDB_SHOW_STABLES_NAME_INDEX, + TSDB_SHOW_STABLES_CREATED_TIME_INDEX, + TSDB_SHOW_STABLES_COLUMNS_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_TID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_MAX_SHOW_STABLES +}; enum _describe_table_index { TSDB_DESCRIBE_METRIC_FIELD_INDEX, TSDB_DESCRIBE_METRIC_TYPE_INDEX, @@ -173,6 +209,7 @@ typedef struct SSuperTable_S { int childTblCount; bool superTblExists; // 0: no, 1: yes bool childTblExists; // 0: no, 1: yes + int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample @@ -218,6 +255,28 @@ typedef struct SSuperTable_S { int64_t totalAffectedRows; } SSuperTable; +typedef struct { + char name[TSDB_DB_NAME_LEN + 1]; + char create_time[32]; + int32_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; +} SDbInfo; + typedef struct SDbCfg_S { // int maxtablesPerVnode; int minRows; @@ -808,13 +867,14 @@ static void init_rand_data() { static void printfInsertMeta() { printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n"); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); - printf("user: \033[33m%s\033[0m\n", g_Dbs.user); - printf("password: \033[33m%s\033[0m\n", g_Dbs.password); - printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); - printf("thread count: \033[33m%d\033[0m\n", g_Dbs.threadCount); - - printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("user: \033[33m%s\033[0m\n", g_Dbs.user); + printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); + printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); + printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + + printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName); @@ -944,11 +1004,12 @@ static void printfInsertMeta() { static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "================ insert.json parse result START================\n"); - fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); - fprintf(fp, "user: %s\n", g_Dbs.user); - fprintf(fp, "password: %s\n", g_Dbs.password); - fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); - fprintf(fp, "thread count: %d\n", g_Dbs.threadCount); + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); + fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "password: %s\n", g_Dbs.password); + fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); + fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); + fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -1123,6 +1184,272 @@ static void printfQueryMeta() { printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); } + +static char* xFormatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + } else { + tt = (time_t)(val / 1000); + } + +/* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + } else { + sprintf(buf + pos, ".%03d", (int)(val % 1000)); + } + + return buf; +} + +static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) { + if (val == NULL) { + fprintf(fp, "%s", TSDB_DATA_NULL_STR); + return; + } + + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + fprintf(fp, "%d", *((int8_t *)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + fprintf(fp, "%d", *((int16_t *)val)); + break; + case TSDB_DATA_TYPE_INT: + fprintf(fp, "%d", *((int32_t *)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + fprintf(fp, "%" PRId64, *((int64_t *)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(buf, val, length); + buf[length] = 0; + fprintf(fp, "\'%s\'", buf); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + xFormatTimestamp(buf, *(int64_t*)val, precision); + fprintf(fp, "'%s'", buf); + break; + default: + break; + } +} + +static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + FILE* fp = fopen(fname, "at"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", fname); + return -1; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + for (int col = 0; col < num_fields; col++) { + if (col > 0) { + fprintf(fp, ","); + } + fprintf(fp, "%s", fields[col].name); + } + fputc('\n', fp); + + int numOfRows = 0; + do { + int32_t* length = taos_fetch_lengths(tres); + for (int i = 0; i < num_fields; i++) { + if (i > 0) { + fputc(',', fp); + } + xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); + } + fputc('\n', fp); + + numOfRows++; + row = taos_fetch_row(tres); + } while( row != NULL); + + fclose(fp); + + return numOfRows; +} + +static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + res = taos_query(taos, "show databases;"); + int32_t code = taos_errno(res); + + if (code != 0) { + fprintf(stderr, "failed to run , reason: %s\n", taos_errstr(res)); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(res); + + while ((row = taos_fetch_row(res)) != NULL) { + // sys database name : 'log' + if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue; + + dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); + if (dbInfos[count] == NULL) { + fprintf(stderr, "failed to allocate memory for some dbInfo[%d]\n", count); + return -1; + } + + strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + xFormatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI); + dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + + strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); + strncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); + + count++; + if (count > MAX_DATABASE_COUNT) { + fprintf(stderr, "The database count overflow than %d\n", MAX_DATABASE_COUNT); + break; + } + } + + return count; +} + +static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) { + FILE *fp = NULL; + if (filename[0] != 0) { + fp = fopen(filename, "at"); + if (fp == NULL) { + fprintf(stderr, "failed to open file: %s\n", filename); + return; + } + } + + fprintf(fp, "================ database[%d] ================\n", index); + fprintf(fp, "name: %s\n", dbInfos->name); + fprintf(fp, "created_time: %s\n", dbInfos->create_time); + fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); + fprintf(fp, "replica: %d\n", dbInfos->replica); + fprintf(fp, "quorum: %d\n", dbInfos->quorum); + fprintf(fp, "days: %d\n", dbInfos->days); + fprintf(fp, "keep1,keep2,keep(D): %s\n", dbInfos->keeplist); + fprintf(fp, "cache(MB): %d\n", dbInfos->cache); + fprintf(fp, "blocks: %d\n", dbInfos->blocks); + fprintf(fp, "minrows: %d\n", dbInfos->minrows); + fprintf(fp, "maxrows: %d\n", dbInfos->maxrows); + fprintf(fp, "wallevel: %d\n", dbInfos->wallevel); + fprintf(fp, "fsync: %d\n", dbInfos->fsync); + fprintf(fp, "comp: %d\n", dbInfos->comp); + fprintf(fp, "cachelast: %d\n", dbInfos->cachelast); + fprintf(fp, "precision: %s\n", dbInfos->precision); + fprintf(fp, "update: %d\n", dbInfos->update); + fprintf(fp, "status: %s\n", dbInfos->status); + fprintf(fp, "\n"); + + fclose(fp); +} + +static void printfQuerySystemInfo(TAOS * taos) { + char filename[MAX_QUERY_SQL_LENGTH+1] = {0}; + char buffer[MAX_QUERY_SQL_LENGTH+1] = {0}; + TAOS_RES* res; + + time_t t; + struct tm* lt; + time(&t); + lt = localtime(&t); + snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec); + + // show variables + res = taos_query(taos, "show variables;"); + //getResult(res, filename); + xDumpResultToFile(filename, res); + + // show dnodes + res = taos_query(taos, "show dnodes;"); + xDumpResultToFile(filename, res); + //getResult(res, filename); + + // show databases + res = taos_query(taos, "show databases;"); + SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *)); + if (dbInfos == NULL) { + fprintf(stderr, "failed to allocate memory\n"); + return; + } + int dbCount = getDbFromServer(taos, dbInfos); + if (dbCount <= 0) return; + + for (int i = 0; i < dbCount; i++) { + // printf database info + printfDbInfoForQueryToFile(filename, dbInfos[i], i); + + // show db.vgroups + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + // show db.stables + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + free(dbInfos[i]); + } + + free(dbInfos); + +} + + #ifdef TD_LOWA_CURL static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -1730,19 +2057,27 @@ static int createDatabases() { void * createTable(void *sarg) -{ - char command[BUFFER_SIZE] = "\0"; - +{ threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; int64_t lastPrintTime = taosGetTimestampMs(); + char* buffer = calloc(superTblInfo->maxSqlLen, 1); + + int len = 0; + int batchNum = 0; //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { if (0 == g_Dbs.use_metric) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); + snprintf(buffer, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); } else { + if (0 == len) { + batchNum = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "create table "); + } + char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); @@ -1750,13 +2085,22 @@ void * createTable(void *sarg) tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { + free(buffer); return NULL; } - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + + len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); free(tagsValBuf); + batchNum++; + + if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } } - - if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){ + + len = 0; + if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){ + free(buffer); return NULL; } @@ -1766,7 +2110,12 @@ void * createTable(void *sarg) lastPrintTime = currentPrintTime; } } - + + if (0 != len) { + (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE); + } + + free(buffer); return NULL; } @@ -2422,6 +2771,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("failed to read json, auto_create_table not found"); goto PARSE_OVER; } + + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); + if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; + } else if (!batchCreateTbl) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 2000; + } else { + printf("failed to read json, batch_create_tbl_num not found"); + goto PARSE_OVER; + } cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { @@ -3679,14 +4038,14 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu b = ntables % threads; } - TAOS* taos; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == taos) { - printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); - exit(-1); - } - } + //TAOS* taos; + //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + // if (NULL == taos) { + // printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); + // exit(-1); + // } + //} int32_t timePrec = TSDB_TIME_PRECISION_MILLI; if (0 != precision[0]) { @@ -3719,7 +4078,12 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu t_info->start_time = start_time; if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - t_info->taos = taos; + //t_info->taos = taos; + t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + if (NULL == t_info->taos) { + printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL)); + exit(-1); + } } else { t_info->taos = NULL; #ifdef TD_LOWA_CURL @@ -3754,6 +4118,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu threadInfo *t_info = infos + i; tsem_destroy(&(t_info->lock_sem)); + taos_close(t_info->taos); superTblInfo->totalAffectedRows += t_info->totalAffectedRows; superTblInfo->totalRowsInserted += t_info->totalRowsInserted; @@ -3766,7 +4131,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu double end = getCurrentTime(); - taos_close(taos); + //taos_close(taos); free(pids); free(infos); @@ -4093,7 +4458,7 @@ void *subQueryProcess(void *sarg) { int queryTestProcess() { TAOS * taos = NULL; taos_init(); - taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, NULL, g_queryInfo.port); if (taos == NULL) { fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); exit(-1); @@ -4106,6 +4471,8 @@ int queryTestProcess() { printfQueryMeta(); printf("Press enter key to continue\n\n"); (void)getchar(); + + printfQuerySystemInfo(taos); pthread_t *pids = NULL; threadInfo *infos = NULL; diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index a853342ab3934544e95198edd05a3684ca3a509b..dbb5f56692341f62378c7d65fa937ee3d20788c4 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -679,7 +679,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb); if (pDb == NULL) break; - if (pDb->pAcct != pUser->pAcct) { + if (pDb->pAcct != pUser->pAcct || pDb->status != TSDB_DB_STATUS_READY) { mnodeDecDbRef(pDb); continue; } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index deba859f94be7374d382b593bab158f222ec5bd4..7f463899ce2b484bcd3978fa26e4c84394cbecd7 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1081,20 +1081,13 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); - } else { - mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); + + return code; } - return code; -} - -static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { - if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, - pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); - if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); while (pVgId) { @@ -1122,6 +1115,16 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mnodeDropAllChildTablesInStable(pStable); } + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { + if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; + mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 2a05d5682e7de5c05beab9fd3f3b124ea85de20c..7c206afe7a12e9be17ad997ee50625a0cce438e9 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -85,6 +85,7 @@ extern "C" { #define TAOS_OS_FUNC_STRING_STR2INT64 #define TAOS_OS_FUNC_SYSINFO #define TAOS_OS_FUNC_TIMER +#define TAOS_OS_FUNC_SEMPHONE_PTHREAD // specific #define htobe64 htonll @@ -114,7 +115,7 @@ int64_t tsosStr2int64(char *str); void taos_block_sigalrm(void); #define TAOS_OS_DEF_EPOLL - #define TAOS_EPOLL_WAIT_TIME 500 + #define TAOS_EPOLL_WAIT_TIME 500 typedef int32_t SOCKET; typedef SOCKET EpollFd; #define EpollClose(pollFd) epoll_close(pollFd) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index 04cb8b6e74b870d73c8f27707bd8a92f7b309e98..cb91b0526bdec17af1d5d86bfb24f8f95b56e01c 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -26,10 +26,6 @@ extern "C" { #endif #endif -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) -#endif - #define FD_VALID(x) ((x) > STDERR_FILENO) #define FD_INITIALIZER ((int32_t)-1) diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 6f96e4d1c80f5f376684f758140be16cc9498b65..d54d519cc3c893885a1c92b04501ee4b558290af 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -46,6 +46,8 @@ #include "msvcFcntl.h" #include "msvcLibgen.h" #include "msvcStdio.h" +#include "msvcUnistd.h" +#include "msvcLibgen.h" #include "sys/msvcStat.h" #include "sys/msvcTypes.h" @@ -144,7 +146,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define in_addr_t unsigned long #define socklen_t int #define htobe64 htonll -#define getpid _getpid struct tm *localtime_r(const time_t *timep, struct tm *result); char * strptime(const char *buf, const char *fmt, struct tm *tm); @@ -153,15 +154,8 @@ char * getpass(const char *prefix); int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); -char * dirname(char *pszPathname); int gettimeofday(struct timeval *ptv, void *pTimeZone); -// for access function in io.h -#define F_OK 00 //Existence only -#define W_OK 02 //Write - only -#define R_OK 04 //Read - only -#define X_OK 06 //Read and write - // for send function in tsocket.c #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 @@ -208,8 +202,6 @@ typedef struct { int wordexp(char *words, wordexp_t *pwordexp, int flags); void wordfree(wordexp_t *pwordexp); -char *realpath(char *path, char *resolved_path); - #define openlog(a, b, c) #define closelog() #define LOG_ERR 0 diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index da4b32139e837a21a4a5a04c129b5a9a50081c5a..83344df22104ccd4e691494089d7364ae647d719 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,17 +17,29 @@ #include "os.h" #include "tglobal.h" +static const char* expand_like_shell(const char *path) { + static __thread char buf[TSDB_FILENAME_LEN]; + buf[0] = '\0'; + wordexp_t we; + if (wordexp(path, &we, 0)) return "/tmp/taosd"; + if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd"; + wordfree(&we); + return buf; +} + void osInit() { if (configDir[0] == 0) { - strcpy(configDir, "~/TDengine/cfg"); + strcpy(configDir, expand_like_shell("~/TDengine/cfg")); } strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(tsDataDir, "~/TDengine/data"); - strcpy(tsLogDir, "~/TDengine/log"); - strcpy(tsScriptDir, "~/TDengine/cfg"); + + strcpy(tsDataDir, expand_like_shell("~/TDengine/data")); + strcpy(tsLogDir, expand_like_shell("~/TDengine/log")); + strcpy(tsScriptDir, expand_like_shell("~/TDengine/cfg")); + strcpy(tsOsName, "Darwin"); } diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 1e77cd68d8476d0184af423300ba21af74de9772..4236ea1c5f08c8b8a8a81ae10c7b442ad3b5ce6a 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -17,71 +17,49 @@ #include "os.h" #include "tulog.h" -#define _SEND_FILE_STEP_ 1000 - int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); - } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } + int r = 0; + if (offset) { + r = fseek(in_file, *offset, SEEK_SET); + if (r==-1) return -1; } - - int remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); - if (rlen <= 0) { - return writeLen; - } else { - fwrite(buffer, 1, remain, out_file); - writeLen += remain; + off_t len = count; + while (len>0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len 0) { - int32_t rlen = read(sfd, buffer, (int32_t)remain); - if (rlen <= 0) { - return writeLen; - } - else { - taosWriteSocket(sfd, buffer, (int32_t)remain); - writeLen += remain; - } + off_t len = count; + while (len>0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len + // #define SEM_USE_PTHREAD // #define SEM_USE_POSIX #define SEM_USE_SEM @@ -279,3 +281,41 @@ int tsem_destroy(tsem_t *sem) { return 0; } +bool taosCheckPthreadValid(pthread_t thread) { + uint64_t id = 0; + int r = pthread_threadid_np(thread, &id); + return r ? false : true; +} + +int64_t taosGetSelfPthreadId() { + return (int64_t)pthread_self(); +} + +int64_t taosGetPthreadId(pthread_t thread) { + return (int64_t)thread; +} + +void taosResetPthread(pthread_t* thread) { + *thread = NULL; +} + +bool taosComparePthread(pthread_t first, pthread_t second) { + return pthread_equal(first, second) ? true : false; +} + +int32_t taosGetPId() { + return (int32_t)getpid(); +} + +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { + char buf[PATH_MAX+1]; + buf[0] = '\0'; + proc_name(getpid(), buf, sizeof(buf)-1); + buf[PATH_MAX] = '\0'; + size_t n = strlen(buf); + if (len) *len = n; + if (name) strcpy(name, buf); + return 0; +} + + diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index bce60429c5efa3358928b74dcd297b473e877587..6af6285f56aa33200379aa52f94e0ab39de80644 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -24,42 +24,134 @@ static void taosGetSystemTimezone() { - // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); + if (cfg_timezone == NULL) return; + if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) { + return; + } + + /* load time zone string from /etc/localtime */ + char buf[4096]; + char *tz = NULL; { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n<0) { + uError("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for(int i=n-1; i>=0; --i) { + if (buf[i]=='/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } } - else { - strcpy(tsTimezone, tz); + if (!tz || 0==strchr(tz, '/')) { + uError("parsing /etc/localtime failed"); + return; } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); + + setenv("TZ", tz, 1); + tzset(); } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", + tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600); + + // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uWarn("timezone not configured, set to system default:%s", tsTimezone); } -static void taosGetSystemLocale() { - // get and set default locale +/* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ +static void taosGetSystemLocale() { // get and set default locale + char sep = '.'; + char *locale = NULL; + SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(tsLocale, "en_US.UTF-8"); + } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); + uWarn("locale not configured, set to system default:%s", tsLocale); } } + /* if user does not specify the charset, extract it from locale */ SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); + char *str = strrchr(tsLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); + + free(revisedCharset); + uWarn("charset not configured, set to system default:%s", tsCharset); + } else { + strcpy(tsCharset, "UTF-8"); + uWarn("can't get locale and charset from system, set it to UTF-8"); + } } } -void taosPrintOsInfo() {} +void taosPrintOsInfo() { + uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize / 1024); + // uInfo(" os openMax: %" PRId64, tsOpenMax); + // uInfo(" os streamMax: %" PRId64, tsStreamMax); + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + + struct utsname buf; + if (uname(&buf)) { + uInfo(" can't fetch os info"); + return; + } + uInfo(" os sysname: %s", buf.sysname); + uInfo(" os nodename: %s", buf.nodename); + uInfo(" os release: %s", buf.release); + uInfo(" os version: %s", buf.version); + uInfo(" os machine: %s", buf.machine); + uInfo("=================================="); +} void taosKillSystem() { uError("function taosKillSystem, exit!"); @@ -67,6 +159,22 @@ void taosKillSystem() { } void taosGetSystemInfo() { + // taosGetProcInfos(); + + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); + long physical_pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); + tsPageSize = page_size; + + // float tmp1, tmp2; + // taosGetSysMemory(&tmp1); + // taosGetProcMemory(&tmp2); + // taosGetDisk(); + // taosGetBandSpeed(&tmp1); + // taosGetCpuUsage(&tmp1, &tmp2); + // taosGetProcIO(&tmp1, &tmp2); + taosGetSystemTimezone(); taosGetSystemLocale(); } @@ -121,7 +229,6 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { char cmdline[1024]; char *taosGetCmdlineByPID(int pid) { - errno = 0; if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { @@ -136,6 +243,7 @@ bool taosGetSystemUid(char *uid) { uuid_t uuid = {0}; uuid_generate(uuid); // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null - uuid_unparse(uuid, uid); + uuid_unparse_lower(uuid, uid); return true; } + diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 42f2ff3afe3c1efe7603af32985c6c52ddc84cd4..0b7b5ca487e5728a5dbdd93e45ee2b0593f98000 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -142,6 +142,8 @@ int64_t taosCopy(char *from, char *to) { if (bytes < sizeof(buffer)) break; } + fsync(fidto); + close(fidfrom); close(fidto); return size; diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index d379e56ed83bd19e4b9ff5336e95cd019f296a36..1d05ce20a5af51b8e941b0a92d2aea0a04b94882 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -62,4 +62,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) { return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 1fb235a00575529be660cea18c1401f1d075a49b..67237e655cd2916ec2ec80daa01e7a0f9ae0d3e3 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -75,18 +75,6 @@ char *getpass(const char *prefix) { return passwd; } -char *strndup(const char *s, size_t n) { - size_t len = strlen(s); - if (len >= n) { - len = n; - } - - char *r = calloc(len + 1, 1); - memcpy(r, s, len); - r[len] = 0; - return r; -} - int twcslen(const wchar_t *wcs) { int *wstr = (int *)wcs; if (NULL == wstr) { diff --git a/src/os/src/windows/wWordexp.c b/src/os/src/windows/wWordexp.c index 929505516dede880e0354dbed63919198ae9e5d1..febe22ac8fa8566607fe2bf9b0be9cc7443c8012 100644 --- a/src/os/src/windows/wWordexp.c +++ b/src/os/src/windows/wWordexp.c @@ -38,7 +38,3 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { } void wordfree(wordexp_t *pwordexp) {} - -char *realpath(char *path, char *resolved_path) { - return _fullpath(path, resolved_path, TSDB_FILENAME_LEN - 1); -} \ No newline at end of file diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 4721451529fb2383b35e15af09c435330fd36aa4..203db218953b6d9859a4e60b589ce01e730e2773 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -92,6 +92,11 @@ void httpStopSystem() { tsHttpServer.stop = 1; #ifdef WINDOWS closesocket(tsHttpServer.fd); +#elif __APPLE__ + if (tsHttpServer.fd!=-1) { + close(tsHttpServer.fd); + tsHttpServer.fd = -1; + } #else shutdown(tsHttpServer.fd, SHUT_RD); #endif diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d0839b3dc4faf18535a1cef58315f5c420128cf1..2efd4f76ea3f3efbbc1c52df4de2e743b65908f3 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -279,7 +279,7 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { pExpr->nSQLOptr = optrType; pExpr->pLeft = pLeft; - if (pRight == NULL) { + if (pLeft != NULL && pRight == NULL) { pRight = calloc(1, sizeof(tSQLExpr)); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 3295c130ddc6060b90338ce60948b4d8ee1297dc..3162ab2e4cbabc2a1ce9ed863589e3be0e92f75f 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -17,7 +17,7 @@ #include "tsocket.h" #include "tutil.h" #include "taosdef.h" -#include "taoserror.h" +#include "taoserror.h" #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" @@ -81,7 +81,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -95,7 +95,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); free(pServerObj); return NULL; } @@ -105,18 +105,18 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - // initialize parameters in case it may encounter error later + // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); for (int j=0; jpThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; } - + pServerObj->pThreadObj[i] = pThreadObj; pThreadObj->pollFd = -1; taosResetPthread(&pThreadObj->thread); @@ -152,9 +152,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->fd < 0) code = -1; + if (pServerObj->fd < 0) code = -1; - if (code == 0) { + if (code == 0) { code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); @@ -162,7 +162,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpTcpServer(pServerObj); pServerObj = NULL; } else { @@ -175,8 +175,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread static void taosStopTcpThread(SThreadObj* pThreadObj) { if (pThreadObj == NULL) { return;} - // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + // save thread into local variable and signal thread to stop + pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } @@ -197,6 +197,11 @@ void taosStopTcpServer(void *handle) { if (pServerObj->fd >= 0) { #ifdef WINDOWS closesocket(pServerObj->fd); +#elif defined(__APPLE__) + if (pServerObj->fd!=-1) { + close(pServerObj->fd); + pServerObj->fd = -1; + } #else shutdown(pServerObj->fd, SHUT_RD); #endif @@ -265,7 +270,7 @@ static void *taosAcceptTcpConnection(void *arg) { taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); continue; } - + // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -274,13 +279,13 @@ static void *taosAcceptTcpConnection(void *arg) { if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); - } + } // pick up next thread for next connection threadId++; @@ -295,17 +300,17 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); if (pClientObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; - } + } tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); tfree(pClientObj); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); } int code = 0; @@ -317,7 +322,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); for (int j=0; jpThreadObj[j]); free(pClientObj); pthread_attr_destroy(&thattr); @@ -356,11 +361,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread pThreadObj->threadId = i; } if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpTcpClient(pClientObj); pClientObj = NULL; - } - return pClientObj; + } + return pClientObj; } void taosStopTcpClient(void *chandle) { @@ -374,20 +379,20 @@ void taosStopTcpClient(void *chandle) { void taosCleanUpTcpClient(void *chandle) { SClientObj *pClientObj = chandle; if (pClientObj == NULL) return; - for (int i = 0; i < pClientObj->numOfThreads; ++i) { + for (int i = 0; i < pClientObj->numOfThreads; ++i) { SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; - taosStopTcpThread(pThreadObj); + taosStopTcpThread(pThreadObj); } - + tDebug("%s TCP client is cleaned up", pClientObj->label); tfree(pClientObj->pThreadObj); - tfree(pClientObj); + tfree(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SClientObj * pClientObj = shandle; int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); + atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); @@ -402,12 +407,12 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin } SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); - + if (pFdObj) { pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); @@ -422,7 +427,7 @@ void taosCloseTcpConnection(void *chandle) { if (pFdObj == NULL || pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); + tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; @@ -435,7 +440,7 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand SThreadObj *pThreadObj = pFdObj->pThreadObj; int ret = taosWriteMsg(pFdObj->fd, data, len); - tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); return ret; } @@ -458,7 +463,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { recvInfo.chandle = NULL; recvInfo.connType = RPC_CONN_TCP; (*(pThreadObj->processData))(&recvInfo); - } + } taosFreeFdObj(pFdObj); } @@ -473,7 +478,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); - return -1; + return -1; } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); @@ -491,14 +496,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } memcpy(msg, &rpcHead, sizeof(SRpcHead)); - + pInfo->msg = msg; pInfo->msgLen = msgLen; pInfo->ip = pFdObj->ip; @@ -509,7 +514,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + free(buffer); return -1; } @@ -557,7 +562,7 @@ static void *taosProcessTcpData(void *param) { } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - shutdown(pFdObj->fd, SHUT_WR); + shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -565,7 +570,7 @@ static void *taosProcessTcpData(void *param) { if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } - if (pThreadObj->stop) break; + if (pThreadObj->stop) break; } if (pThreadObj->pollFd >=0) { @@ -603,7 +608,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -637,7 +642,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); if (pFdObj->prev) { @@ -652,7 +657,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 22bdc7e74e2ba08b4ed297bd02901d4411835350..3ad9e9bba01eb861c48fa121b154af3e7838dda0 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -103,6 +103,11 @@ void syncCloseTcpThreadPool(void *param) { #ifdef WINDOWS closesocket(pPool->acceptFd); +#elif defined(__APPLE__) + if (pPool->acceptFd!=-1) { + close(pPool->acceptFd); + pPool->acceptFd = -1; + } #else shutdown(pPool->acceptFd, SHUT_RD); #endif diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index d942151843ae4573b6cfa3f65bdc315919364923..7b7c9b6127db0f1c8cabab35254ba4a128e05f05 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -593,7 +593,7 @@ void taosGetDisk() { tsAvailLogDirGB = (float)(diskSize.avail / unit); } - if (taosGetDiskSize("/tmp", &diskSize) == 0) { + if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { tsTotalTmpDirGB = (float)(diskSize.tsize / unit); tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); } diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index 132e90f8d1fdaad8812c34d872cca19f79d417df..f1e2422e45b195e3874518132051ded67a776449 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -20,6 +20,8 @@ #define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TSDB_IVLD_FID INT_MIN +#define TSDB_FILE_STATE_OK 0 +#define TSDB_FILE_STATE_BAD 1 #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) @@ -31,6 +33,10 @@ #define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf)) #define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf)) #define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf)) +#define TSDB_FILE_STATE(tf) ((tf)->state) +#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) +#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; @@ -47,10 +53,11 @@ typedef struct { SMFInfo info; TFILE f; int fd; + uint8_t state; } SMFile; void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); -void tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile); +void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); int tsdbEncodeSMFile(void** buf, SMFile* pMFile); void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile); @@ -165,6 +172,7 @@ typedef struct { SDFInfo info; TFILE f; int fd; + uint8_t state; } SDFile; void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -346,4 +354,14 @@ static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, *maxKey = *minKey + days * tsMsPerDay[precision] - 1; } +static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { + return false; + } + } + + return true; +} + #endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 3216b18459bb8a7028c54ff5cf7f17be81c2e154..a777b11186f064436137d267d73f9a70b5aa8701 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -52,7 +52,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo); static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); static int tsdbCommitTSData(STsdbRepo *pRepo); -static int tsdbStartCommit(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo, int eno); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbCreateCommitIters(SCommitH *pCommith); @@ -84,10 +84,7 @@ static int tsdbApplyRtn(STsdbRepo *pRepo); static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); void *tsdbCommitData(STsdbRepo *pRepo) { - if (tsdbStartCommit(pRepo) < 0) { - tsdbError("vgId:%d failed to commit data while startting to commit since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } + tsdbStartCommit(pRepo); // Commit to update meta file if (tsdbCommitMeta(pRepo) < 0) { @@ -138,11 +135,15 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); } else { tsdbInitMFileEx(&mf, pOMFile); if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } } @@ -154,12 +155,20 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { if (pAct->act == TSDB_UPDATE_META) { pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } } else if (pAct->act == TSDB_DROP_META) { if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } } else { @@ -168,6 +177,9 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { } if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache return -1; } @@ -208,6 +220,8 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); } static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { @@ -238,7 +252,7 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { - pMFile->info.tombSize += pRecord->size; + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); } else { pMFile->info.nRecords++; } @@ -253,7 +267,7 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); if (pRecord == NULL) { - tsdbError("failed to drop KV store record with key %" PRIu64 " since not find", uid); + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); return -1; } @@ -264,11 +278,11 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { void *pBuf = buf; tsdbEncodeKVRecord(&pBuf, &rInfo); - if (tsdbAppendMFile(pMFile, buf, POINTER_DISTANCE(pBuf, buf), NULL) < 0) { + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { return -1; } - pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf)); + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); pMFile->info.nDels++; pMFile->info.nRecords--; pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); @@ -302,7 +316,12 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { // Skip expired memory data and expired FSET tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid >= commith.rtn.minFid) break; + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + break; + } } // Loop to commit to each file @@ -349,7 +368,7 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) { return 0; } -static int tsdbStartCommit(STsdbRepo *pRepo) { +static void tsdbStartCommit(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); @@ -360,7 +379,6 @@ static int tsdbStartCommit(STsdbRepo *pRepo) { tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); pRepo->code = TSDB_CODE_SUCCESS; - return 0; } static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { @@ -413,14 +431,18 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { if (pIter->pTable == NULL) continue; if (tsdbCommitToTable(pCommith, tid) < 0) { - // TODO: revert the file change tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } } if (tsdbWriteBlockIdx(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } @@ -674,7 +696,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { TSDB_RUNLOCK_TABLE(pIter->pTable); - if (tsdbWriteBlockInfo(pCommith) < 0) return -1; + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } return 0; } @@ -926,6 +952,8 @@ static int tsdbWriteBlockIdx(SCommitH *pCommih) { if (nidx <= 0) { // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; return 0; } @@ -1227,7 +1255,6 @@ static void tsdbResetCommitFile(SCommitH *pCommith) { } static void tsdbResetCommitTable(SCommitH *pCommith) { - tdResetDataCols(pCommith->pDataCols); taosArrayClear(pCommith->aSubBlk); taosArrayClear(pCommith->aSupBlk); pCommith->pTable = NULL; @@ -1256,6 +1283,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbCloseAndUnsetFSet(&(pCommith->readh)); return -1; } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { pCommith->isRFileSet = false; } @@ -1266,6 +1296,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); } @@ -1274,6 +1306,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isDFileSame = false; pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); } else { did.level = TSDB_FSET_LEVEL(pSet); did.id = TSDB_FSET_ID(pSet); @@ -1285,6 +1320,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); return -1; @@ -1296,7 +1334,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); tsdbInitDFileEx(pWDataf, pRDataf); if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { - tsdbCloseDFile(pWHeadf); + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); @@ -1313,6 +1354,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isLFileSame = true; if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { @@ -1325,6 +1369,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isLFileSame = false; if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + tsdbCloseDFileSet(pWSet); tsdbRemoveDFile(pWHeadf); if (pCommith->isRFileSet) { @@ -1360,7 +1407,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p if (pBlock->last) { if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; } else { - if (mergeRows < pCfg->maxRowsPerFileBlock) return true; + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; } } @@ -1373,12 +1420,16 @@ static int tsdbApplyRtn(STsdbRepo *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); SDFileSet *pSet; - // Get retentioni snapshot + // Get retention snapshot tsdbGetRtnSnap(pRepo, &rtn); tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); while ((pSet = tsdbFSIterNext(&fsiter))) { - if (pSet->fid < rtn.minFid) continue; + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { return -1; @@ -1392,10 +1443,13 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { SDiskID did; SDFileSet nSet; STsdbFS * pfs = REPO_FS(pRepo); + int level; ASSERT(pSet->fid >= pRtn->minFid); - tfsAllocDisk(tsdbGetFidLevel(pSet->fid, pRtn), &(did.level), &(did.id)); + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); if (did.level == TFS_UNDECIDED_LEVEL) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; @@ -1406,12 +1460,17 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); return -1; } if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { return -1; } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); } else { // On a correct level if (tsdbUpdateDFileSet(pfs, pSet) < 0) { diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index b3b26bcf1fbbd8fdf5b4ef6894bab2a6c5947ac7..e6815083371693a5552372bcc792e14213ae3456 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -701,6 +701,8 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { int64_t maxBufSize = 0; SMFInfo minfo; + taosHashEmpty(pfs->metaCache); + // No meta file, just return if (pfs->cstatus->pmf == NULL) return 0; @@ -718,6 +720,12 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { while (true) { int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; + + if (tsize < 0) { + tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + if (tsize < sizeof(SKVRecord)) { tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord), TSDB_FILE_FULL_NAME(pMFile)); @@ -840,7 +848,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) { continue; } - if (tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { + if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { continue; } diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 9a53bf45779ed3d609e5f5b1e6357ad0d72ad2bb..8124a0e3b5984eecee1f0934a385be56d5bd3b61 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -33,7 +33,7 @@ static int tsdbRollBackDFile(SDFile *pDFile); void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { char fname[TSDB_FILENAME_LEN]; - TSDB_FILE_SET_CLOSED(pMFile); + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); memset(&(pMFile->info), 0, sizeof(pMFile->info)); pMFile->info.magic = TSDB_FILE_INIT_MAGIC; @@ -42,7 +42,7 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); } -void tsdbInitMFileEx(SMFile *pMFile, SMFile *pOMFile) { +void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { *pMFile = *pOMFile; TSDB_FILE_SET_CLOSED(pMFile); } @@ -201,6 +201,7 @@ int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) { tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile)); pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); return 0; } @@ -232,6 +233,7 @@ int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) { tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size); pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return 0; } else { @@ -293,6 +295,8 @@ static int tsdbRollBackMFile(SMFile *pMFile) { void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) { char fname[TSDB_FILENAME_LEN]; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK); + TSDB_FILE_SET_CLOSED(pDFile); memset(&(pDFile->info), 0, sizeof(pDFile->info)); @@ -439,6 +443,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) { tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); return 0; } @@ -470,6 +475,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) { tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return 0; } else { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0931b6281be286aa6d0ac35b942beac8fca98cb3..73a127079920bcdf0cdc2d3e244998310c4d8dbc 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -216,11 +216,13 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { } int tsdbAsyncCommit(STsdbRepo *pRepo) { - if (pRepo->mem == NULL) return 0; - tsem_wait(&(pRepo->readyToCommit)); ASSERT(pRepo->imem == NULL); + if (pRepo->mem == NULL) { + tsem_post(&(pRepo->readyToCommit)); + return 0; + } if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a8376fc824ecc0b76e689b2088fabc6823a87b8f..b16337ed7a6537f341e53b7d47b4867ea0c011cd 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1176,13 +1176,18 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity } static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, - int32_t numOfCols, STable* pTable) { + int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; // the schema version info is embeded in SDataRow - STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - int32_t numOfRowCols = schemaNCols(pSchema); - + int32_t numOfRowCols = 0; + if (pSchema == NULL) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + numOfRowCols = schemaNCols(pSchema); + } else { + numOfRowCols = schemaNCols(pSchema); + } + int32_t i = 0, j = 0; while(i < numOfCols && j < numOfRowCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -1199,10 +1204,38 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if (pSchema->columns[j].colId == pColInfo->info.colId) { void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - memcpy(pData, value, varDataTLen(value)); - } else { - memcpy(pData, value, pColInfo->info.bytes); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + default: + memcpy(pData, value, pColInfo->info.bytes); } j++; @@ -1401,6 +1434,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int16_t rv = -1; + STSchema* pSchema = NULL; + int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; @@ -1429,7 +1465,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1442,7 +1483,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1987,6 +2033,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; + int16_t rv = -1; + STSchema* pSchema = NULL; do { SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); @@ -2007,7 +2055,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2090,7 +2142,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return false; } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); tfree(pRow); // update the last key value @@ -2164,7 +2216,7 @@ bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) { return false; } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); tfree(pRow); // update the last key value diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index bae4637d77775b03d64a7111a4570ef86a686f7c..88ab973f5ed85589996b2a90018036a51df48897 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -85,6 +85,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { pRepo->state = TSDB_STATE_OK; tsdbInitSyncH(&synch, pRepo, socketFd); + tsem_wait(&(pRepo->readyToCommit)); tsdbStartFSTxn(pRepo, 0, 0); if (tsdbSyncRecvMeta(&synch) < 0) { @@ -98,6 +99,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { } tsdbEndFSTxn(pRepo); + tsem_post(&(pRepo->readyToCommit)); tsdbDestroySyncH(&synch); // Reload file change @@ -107,6 +109,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { _err: tsdbEndFSTxnWithError(REPO_FS(pRepo)); + tsem_post(&(pRepo->readyToCommit)); tsdbDestroySyncH(&synch); return -1; } @@ -191,7 +194,8 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { return 0; } - if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0) { + if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0 || + TSDB_FILE_IS_BAD(pLMFile)) { // Local has no meta file or has a different meta file, need to copy from remote pSynch->mfChanged = true; @@ -409,7 +413,8 @@ static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { pSynch->pdf != NULL ? pSynch->pdf->fid : -1); pLSet = tsdbFSIterNext(&fsiter); } else { - if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf)) { + if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && + tsdbFSetIsOk(pLSet)) { // Just keep local files and notify remote not to send tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index eb96f81b3349bea1a67473843de2432b6b474d66..7a92750f8f2e89a93557d367b0c4772a2faaa57d 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -134,6 +134,11 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { wordfree(&full_path); + char tmp[1025] = {0}; + if (realpath(option, tmp) != NULL) { + strcpy(option, tmp); + } + int code = taosMkDir(option, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index ebae8ece14e2f3bd045fee92482afa4c6c403949..7d64bd1f83fb8d235c825057251a5e76e0b96b2a 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -24,22 +24,22 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; // previous node + struct SRefNode *prev; // previous node struct SRefNode *next; // next node - void *p; // pointer to resource protected, + void *p; // pointer to resource protected, int64_t rid; // reference ID int32_t count; // number of references - int removed; // 1: removed + int removed; // 1: removed } SRefNode; typedef struct { SRefNode **nodeList; // array of SRefNode linked list int state; // 0: empty, 1: active; 2: deleted - int rsetId; // refSet ID, global unique + int rsetId; // refSet ID, global unique int64_t rid; // increase by one for each new reference - int max; // mod + int max; // mod int32_t count; // total number of SRefNodes in this set - int64_t *lockedBy; + int64_t *lockedBy; void (*fp)(void *); } SRefSet; @@ -62,9 +62,9 @@ int taosOpenRef(int max, void (*fp)(void *)) SRefSet *pSet; int64_t *lockedBy; int i, rsetId; - + pthread_once(&tsRefModuleInit, taosInitRefModule); - + nodeList = calloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -79,12 +79,12 @@ int taosOpenRef(int max, void (*fp)(void *)) } pthread_mutex_lock(&tsRefMutex); - + for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; - } + } if (i < TSDB_REF_OBJECTS) { rsetId = tsNextId; @@ -105,7 +105,7 @@ int taosOpenRef(int max, void (*fp)(void *)) free (nodeList); free (lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); - } + } pthread_mutex_unlock(&tsRefMutex); @@ -127,7 +127,7 @@ int taosCloseRef(int rsetId) pthread_mutex_lock(&tsRefMutex); - if (pSet->state == TSDB_REF_STATE_ACTIVE) { + if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; deleted = 1; uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count); @@ -142,7 +142,7 @@ int taosCloseRef(int rsetId) return 0; } -int64_t taosAddRef(int rsetId, void *p) +int64_t taosAddRef(int rsetId, void *p) { int hash; SRefNode *pNode; @@ -163,7 +163,7 @@ int64_t taosAddRef(int rsetId, void *p) terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + pNode = calloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -173,7 +173,7 @@ int64_t taosAddRef(int rsetId, void *p) rid = atomic_add_fetch_64(&pSet->rid, 1); hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode->p = p; pNode->rid = rid; pNode->count = 1; @@ -187,16 +187,16 @@ int64_t taosAddRef(int rsetId, void *p) taosUnlockList(pSet->lockedBy+hash); - return rid; + return rid; } -int taosRemoveRef(int rsetId, int64_t rid) +int taosRemoveRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosAcquireRef(int rsetId, int64_t rid) +void *taosAcquireRef(int rsetId, int64_t rid) { int hash; SRefNode *pNode; @@ -204,7 +204,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + //uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } @@ -223,7 +223,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) terrno = TSDB_CODE_REF_ID_REMOVED; return NULL; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); @@ -233,8 +233,8 @@ void *taosAcquireRef(int rsetId, int64_t rid) if (pNode->rid == rid) { break; } - - pNode = pNode->next; + + pNode = pNode->next; } if (pNode) { @@ -258,7 +258,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) return p; } -int taosReleaseRef(int rsetId, int64_t rid) +int taosReleaseRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } @@ -280,6 +280,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } + void *newP = NULL; pSet = tsRefSetList + rsetId; taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { @@ -289,52 +290,68 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } - int hash = 0; - if (rid > 0) { - hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + do { + newP = NULL; + int hash = 0; + if (rid > 0) { + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) break; + pNode = pNode->next; + } - pNode = pSet->nodeList[hash]; - while (pNode) { - if (pNode->rid == rid) break; - pNode = pNode->next; - } + if (pNode == NULL) { + uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + taosUnlockList(pSet->lockedBy+hash); + taosDecRsetCount(pSet); + return NULL; + } - if (pNode == NULL) { - uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); - terrno = TSDB_CODE_REF_NOT_EXIST; - taosUnlockList(pSet->lockedBy+hash); - return NULL; + // rid is there + pNode = pNode->next; + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode == NULL) { + taosUnlockList(pSet->lockedBy+hash); + hash++; + } } - // rid is there - pNode = pNode->next; - if (pNode == NULL) { - taosUnlockList(pSet->lockedBy+hash); - hash++; + if (pNode == NULL) { + for (; hash < pSet->max; ++hash) { + taosLockList(pSet->lockedBy+hash); + pNode = pSet->nodeList[hash]; + if (pNode) { + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode) break; + } + taosUnlockList(pSet->lockedBy+hash); + } } - } - if (pNode == NULL) { - for (; hash < pSet->max; ++hash) { - taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - if (pNode) break; + if (pNode) { + pNode->count++; // acquire it + newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + } else { + uTrace("rsetId:%d the list is over", rsetId); } - } - void *newP = NULL; - if (pNode) { - pNode->count++; // acquire it - newP = pNode->p; - taosUnlockList(pSet->lockedBy+hash); - uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); - } else { - uTrace("rsetId:%d the list is over", rsetId); - } - - if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (pNode) rid = pNode->rid; + } while (newP && pNode->removed); taosDecRsetCount(pSet); @@ -350,22 +367,22 @@ int taosListRef() { for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; - - if (pSet->state == TSDB_REF_STATE_EMPTY) + + if (pSet->state == TSDB_REF_STATE_EMPTY) continue; uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); for (int j=0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; - + while (pNode) { uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count); pNode = pNode->next; num++; } - } - } + } + } pthread_mutex_unlock(&tsRefMutex); @@ -397,16 +414,16 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode = pSet->nodeList[hash]; while (pNode) { if (pNode->rid == rid) break; - pNode = pNode->next; + pNode = pNode->next; } if (pNode) { @@ -417,18 +434,18 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode->prev) { pNode->prev->next = pNode->next; } else { - pSet->nodeList[hash] = pNode->next; + pSet->nodeList[hash] = pNode->next; } if (pNode->next) { - pNode->next->prev = pNode->prev; - } + pNode->next->prev = pNode->prev; + } released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; code = -1; } @@ -437,11 +454,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (released) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - (*pSet->fp)(pNode->p); + (*pSet->fp)(pNode->p); free(pNode); taosDecRsetCount(pSet); - } + } return code; } @@ -493,5 +510,5 @@ static void taosDecRsetCount(SRefSet *pSet) { } pthread_mutex_unlock(&tsRefMutex); -} +} diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 7ccdca0fb1f44a5c9628ad718c1abf41f7a10df0..e07587686774cb0fa6c7682daca59b400a9451ec 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -32,14 +32,27 @@ int32_t taosGetFqdn(char *fqdn) { struct addrinfo hints = {0}; struct addrinfo *result = NULL; +#ifdef __APPLE__ + // on macosx, hostname -f has the form of xxx.local + // which will block getaddrinfo for a few seconds if AI_CANONNAME is set + // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return + // immediately + hints.ai_family = AF_INET; +#else // __APPLE__ hints.ai_flags = AI_CANONNAME; +#endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } +#ifdef __APPLE__ + // refer to comments above + strcpy(fqdn, hostname); +#else // __APPLE__ strcpy(fqdn, result->ai_canonname); +#endif // __APPLE__ freeaddrinfo(result); return 0; } @@ -286,7 +299,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie int32_t bufSize = 1024 * 1024; sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - + if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 809b69e8ad73fd06af18a2a52593a3913d603491..98334497772d60991a553b5d929404ce5e3437e4 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -188,11 +188,7 @@ static void removeTimer(uintptr_t id) { } static int64_t getMonotonicMs(void) { -#ifdef WINDOWS return (int64_t) getMonotonicUs() / 1000; -#else - return taosGetTimestampMs(); -#endif } static void addToWheel(tmr_obj_t* timer, uint32_t delay) { diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 9b7916c8bd3b4c5878b0edc3431802b3ed7cd4f6..03f2b11eec239ad469faec66ddcbd60282e0409b 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -34,6 +34,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.update = vnodeMsg->cfg.update; pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; @@ -227,6 +228,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *update = cJSON_GetObjectItem(root, "update"); + if (!update || update->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file); + vnodeMsg.cfg.update = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.update = (int8_t)update->valueint; + } + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); @@ -325,6 +335,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"update\": %d,\n", pMsg->cfg.update); len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index cd487105b8c74ec8277dfb5fafb5914d6323e2fb..ac9536d243dbd2613d42c8b9a5b054ce862df061 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -194,12 +194,14 @@ int32_t vnodeOpen(int32_t vgId) { int32_t code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId); vnodeCleanUp(pVnode); - return code; + return 0; } code = vnodeReadVersion(pVnode); if (code != TSDB_CODE_SUCCESS) { + pVnode->version = 0; vError("vgId:%d, failed to read file version, generate it from data file", pVnode->vgId); // Allow vnode start even when read file version fails, set file version as wal version or zero // vnodeCleanUp(pVnode); diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 20ab3951586d353909090cb1344fad10d0362ddf..de7c5989d1ecdaec578e4322ac49491587ad851b 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include #define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) #define A(statement, fmt, ...) do { \ @@ -56,6 +58,8 @@ ##__VA_ARGS__); \ } while (0) +#include "os.h" + typedef struct ep_s ep_t; struct ep_s { int ep; @@ -214,7 +218,6 @@ static void* routine(void* arg) { A(0==pthread_mutex_lock(&ep->lock), ""); ep->wakenup = 0; A(0==pthread_mutex_unlock(&ep->lock), ""); - D("........"); continue; } A(ev->data.ptr, "internal logic error"); diff --git a/tests/pytest/functions/function_stddev_td2555.py b/tests/pytest/functions/function_stddev_td2555.py new file mode 100644 index 0000000000000000000000000000000000000000..a4da7d5e29b53ac9051dc39aa8af0708197dacd2 --- /dev/null +++ b/tests/pytest/functions/function_stddev_td2555.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 100 + self.ts = 1537146000000 + self.clist1 = [] + self.clist2 = [] + self.clist3 = [] + self.clist4 = [] + self.clist5 = [] + self.clist6 = [] + + def getData(self): + for i in range(tdSql.queryRows): + for j in range(6): + exec('self.clist{}.append(tdSql.queryResult[i][j+1])'.format(j+1)) + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(cid int,gbid binary(20),loc nchar(20))''') + tdSql.execute("create table test1 using test tags(1,'beijing','北京')") + tdSql.execute("create table test2 using test tags(2,'shanghai','深圳')") + tdSql.execute("create table test3 using test tags(2,'shenzhen','深圳')") + tdSql.execute("create table test4 using test tags(1,'shanghai','上海')") + for j in range(4): + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(now-%dh, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (j+1,i, i + 1, i + 1, i + 1, i + 1, i + i * 0.1, i * 1.5, i % 2, i + 1, i + 1)) + + # stddev verifacation + tdSql.error("select stddev(ts) from test") + tdSql.error("select stddev(col7) from test") + tdSql.error("select stddev(col8) from test") + tdSql.error("select stddev(col9) from test") + + con_list = [ + ' where cid = 1 and ts >=now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts now-1d') - self.checkRows(1,cmd) + self.checkRows(2,cmd) def stop(self): os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index dde9f78953766d41bb05fa5639b3ca4e18f4ef2a..4ec517a0bf1c5eff8ad670cf28ab63d5ce818460 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -1,18 +1,18 @@ # update -#python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update.py python3 ./test.py -f update/allow_update-0.py python3 ./test.py -f update/append_commit_data.py python3 ./test.py -f update/append_commit_last-0.py python3 ./test.py -f update/append_commit_last.py -#python3 ./test.py -f update/merge_commit_data.py -#python3 ./test.py -f update/merge_commit_data-0.py -#python3 ./test.py -f update/merge_commit_data2.py -#python3 ./test.py -f update/merge_commit_data2_update0.py -#python3 ./test.py -f update/merge_commit_last-0.py -#python3 ./test.py -f update/merge_commit_last.py -#python3 ./test.py -f update/bug_td2279.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data-0.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal python3 ./test.py -f wal/addOldWalTest.py diff --git a/tests/pytest/pytest_3.sh b/tests/pytest/pytest_3.sh index a64c144ffd0d2863ab39792e8162694168f397a7..5e59bb879b1a6920dff536b49a13d976eba94256 100755 --- a/tests/pytest/pytest_3.sh +++ b/tests/pytest/pytest_3.sh @@ -99,7 +99,7 @@ python3 test.py -f query/queryFillTest.py python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py -python3 test.py -f tools/taosdemoTest2.py +#python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index 756aa0eda998aaa0aad30e94b2a7192d7d53bb65..8cec38780e5e873cb07d8ade45ef56b5d4ee43f7 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -111,6 +111,18 @@ class TDTestCase: tdSql.query("select * from tb where c5 = 'true' ") tdSql.checkRows(5) + # For jira: https://jira.taosdata.com:18080/browse/TD-2850 + tdSql.execute("create database 'Test' ") + tdSql.execute("use 'Test' ") + tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)") + tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + tdSql.query("select * from tb0") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py index 3d37e6726c965a8963f728fdd1d8dc89e1cee339..0ef816da8d02fc6dcb48954435aad51342248e56 100644 --- a/tests/pytest/stable/insert.py +++ b/tests/pytest/stable/insert.py @@ -47,6 +47,40 @@ class TDTestCase: tdSql.query("select * from db.st where dev='dev_02'") tdSql.checkRows(1) + #For: https://jira.taosdata.com:18080/browse/TD-2671 + print("==============step3") + tdSql.execute( + "create stable if not exists stb (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_01 using stb tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_02 using stb tags("dev_02")') + + print("==============step4") + + tdSql.execute( + """INSERT INTO dev_01(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_02 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.stb where dev='dev_01'") + tdSql.checkRows(2) + + tdSql.query("select * from db.stb where dev='dev_02'") + tdSql.checkRows(1) + + tdSql.query("describe db.stb") + tdSql.checkRows(3) + + tdSql.execute("alter stable db.stb add tag t1 int") + tdSql.query("describe db.stb") + tdSql.checkRows(4) + + tdSql.execute("drop stable db.stb") + tdSql.query("show stables") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py index 7d5627be4363467e5e736760eea1e9f0799437d9..4d7e871e66b87c4c3ff8c46f68f00426d117d956 100644 --- a/tests/pytest/tools/taosdemoTest2.py +++ b/tests/pytest/tools/taosdemoTest2.py @@ -31,10 +31,18 @@ class TDTestCase: def insertDataAndAlterTable(self, threadID): if(threadID == 0): - os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords)) + os.system("yes | taosdemo -t %d -n %d -x" % (self.numberOfTables, self.numberOfRecords)) if(threadID == 1): print("use test") tdSql.execute("use test") + while True: + print("query started") + tdSql.query("select * from test.t9") + rows = tdSql.queryRows + print("rows %d" % rows) + if(rows > 0): + break + time.sleep(1) print("alter table test.meters add column f4 int") tdSql.execute("alter table test.meters add column f4 int") print("insert into test.t0 values (now, 1, 2, 3, 4)") @@ -46,8 +54,8 @@ class TDTestCase: t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, )) t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, )) - t1.start() - time.sleep(2) + t1.start() + time.sleep(2) t2.start() t1.join() t2.join() diff --git a/tests/script/general/parser/dbtbnameValidate.sim b/tests/script/general/parser/dbtbnameValidate.sim index 48c5d4a1f94337a6324826d78eda3ad498ac3d76..5fc67334c4b03329a7807c448eea38ebb4f2bdce 100644 --- a/tests/script/general/parser/dbtbnameValidate.sim +++ b/tests/script/general/parser/dbtbnameValidate.sim @@ -12,24 +12,24 @@ sql create database 'abc123' sql create database '_ab1234' sql create database 'ABC123' sql create database '_ABC123' -sql create database 'aABb123 ' -sql create database ' xyz ' -sql create database ' XYZ ' +sql_error create database 'aABb123 ' +sql_error create database ' xyz ' +sql_error create database ' XYZ ' sql use 'abc123' sql use '_ab1234' sql use 'ABC123' sql use '_ABC123' -sql use 'aABb123' -sql use ' xyz ' -sql use ' XYZ ' +sql_error use 'aABb123' +sql_error use ' xyz ' +sql_error use ' XYZ ' sql drop database 'abc123' sql drop database '_ab1234' sql_error drop database 'ABC123' sql drop database '_ABC123' -sql drop database 'aABb123' -sql drop database ' xyz ' +sql_error drop database 'aABb123' +sql_error drop database ' xyz ' sql_error drop database ' XYZ ' diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 0b5cdda52787971c115a96c3674689f4a42504fc..9ed58a90d1ba6bd781d306070e03f56b75984093 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -44,10 +44,10 @@ echo serverPort 7100 >> %TAOS_CFG% echo logDir %LOG_DIR% >> %TAOS_CFG% echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% echo numOfLogLines 100000000 >> %TAOS_CFG% -echo rpcDebugFlag 143 >> %TAOS_CFG% +echo rpcDebugFlag 135 >> %TAOS_CFG% echo tmrDebugFlag 131 >> %TAOS_CFG% -echo cDebugFlag 143 >> %TAOS_CFG% -echo udebugFlag 143 >> %TAOS_CFG% +echo cDebugFlag 135 >> %TAOS_CFG% +echo udebugFlag 135 >> %TAOS_CFG% echo wal 0 >> %TAOS_CFG% echo asyncLog 0 >> %TAOS_CFG% echo locale en_US.UTF-8 >> %TAOS_CFG% diff --git a/tests/test-all.sh b/tests/test-all.sh index 13efbec7a260aed074a74319b616549c14c14f3e..2374750be461e7f19e0e154a3ed4461506e6a35a 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -6,15 +6,15 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -function git_branch { - branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`" - if [ "${branch}" != "" ];then - if [ "${branch}" = "(no branch)" ];then - branch="(`git rev-parse --short HEAD`...)" - fi - branch=(${branch////_}) - echo "$branch" - fi + +function dohavecore(){ + corefile=`find $corepath -mmin 1` + if [ -n "$corefile" ];then + echo 'taosd or taos has generated core' + if [[ $1 == 1 ]];then + exit 8 + fi + fi } function runSimCaseOneByOne { while read -r line; do @@ -42,6 +42,7 @@ function runSimCaseOneByOne { # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore 0 fi done < $1 } @@ -69,14 +70,15 @@ function runSimCaseOneByOnefq { out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then - cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"` + cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` else - cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi exit 8 fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore 1 fi done < $1 } @@ -105,6 +107,7 @@ function runPyCaseOneByOne { else $line > /dev/null 2>&1 fi + dohavecore 0 fi done < $1 } @@ -126,13 +129,14 @@ function runPyCaseOneByOnefq { end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then - cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` exit 8 fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log else $line > /dev/null 2>&1 fi + dohavecore 1 fi done < $1 } @@ -140,7 +144,7 @@ totalFailed=0 totalPyFailed=0 tests_dir=`pwd` - +corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` if [ "$2" != "python" ]; then echo "### run TSIM test case ###" cd $tests_dir/script