Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 e7a52dfb 编写于 作者: H Haojun Liao
浏览文件
操作

[td-225]merge develop

上级 2ee22867 0bdecede
展开全部 隐藏空白更改
内联 并排
Showing with 1698 addition and 849 deletion
Jenkinsfile
浏览文件 @ e7a52dfb
......@@ -45,7 +45,7 @@ def pre_test(){
git pull
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0
git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|.*src/connector|Jenkinsfile' || exit 0
cd ${WK}
git reset --hard HEAD~10
git checkout develop
......
deps/MsvcLibX/include/msvcUnistd.h
浏览文件 @ e7a52dfb
......@@ -89,11 +89,12 @@ pid_t getppid(void); /* Get parent PID */
/* Path management */
#if defined(_WIN32)
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define realpath realpathU
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
// #define realpath realpathU
#define CompactPath CompactPathU
#else /* _ANSI_SOURCE */
#define realpath realpathA
// #define realpath realpathA
#define CompactPath CompactPathA
#endif
#endif /* defined(_WIN32) */
......
deps/rmonotonic/src/monotonic.c
浏览文件 @ e7a52dfb
......@@ -84,11 +84,11 @@ static void monotonicInit_x86linux() {
regfree(&constTscRegex);
if (mono_ticksPerMicrosecond == 0) {
fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate");
//fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate");
return;
}
if (!constantTsc) {
fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present");
//fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present");
return;
}
......
documentation20/webdocs/markdowndocs/connector-ch.md
浏览文件 @ e7a52dfb
......@@ -852,7 +852,7 @@ npm install td2.0-connector
### Linux
- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
- `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。
- `node` 2.0.6支持v12.x和v10.x，2.0.5及更早版本支持v10.x版本，其他版本可能存在包兼容性的问题。
- `make`
- c语言编译器比如<a href="https://gcc.gnu.org">GCC</a>
......
src/client/src/tscUtil.c
浏览文件 @ e7a52dfb
......@@ -1433,9 +1433,9 @@ int32_t tscValidateName(SStrToken* pToken) {
if (sep == NULL) { // single part
if (pToken->type == TK_STRING) {
strdequote(pToken->z);
tscDequoteAndTrimToken(pToken);
tscStrToLower(pToken->z, pToken->n);
pToken->n = (uint32_t)strtrim(pToken->z);
//pToken->n = (uint32_t)strtrim(pToken->z);
int len = tSQLGetToken(pToken->z, &pToken->type);
......
src/common/src/tglobal.c
浏览文件 @ e7a52dfb
......@@ -59,7 +59,6 @@ char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
int8_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
......@@ -182,6 +181,7 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDataDir[TSDB_FILENAME_LEN] = {0};
char tsScriptDir[TSDB_FILENAME_LEN] = {0};
char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
int32_t tsDiskCfgNum = 0;
......
src/connector/jdbc/pom.xml
浏览文件 @ e7a52dfb
......@@ -36,6 +36,7 @@
<maven-compiler-plugin.version>3.6.0</maven-compiler-plugin.version>
<commons-logging.version>1.1.2</commons-logging.version>
<commons-lang3.version>3.5</commons-lang3.version>
<maven.test.jvmargs></maven.test.jvmargs>
</properties>
<dependencies>
<dependency>
......@@ -122,11 +123,14 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12.4</version>
<configuration>
<forkMode>pertest</forkMode>
<argLine>${maven.test.jvmargs}</argLine>
<includes>
<include>**/*Test.java</include>
</includes>
<excludes>
<exclude>**/AppMemoryLeakTest.java</exclude>
<exclude>**/TaosInfoMonitorTest.java</exclude>
<exclude>**/FailOverTest.java</exclude>
</excludes>
<testFailureIgnore>true</testFailureIgnore>
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java
浏览文件 @ e7a52dfb
......@@ -19,12 +19,12 @@ import java.util.Map;
public abstract class TSDBConstants {
public static final String STATEMENT_CLOSED = "Statement already closed.";
public static final String DEFAULT_PORT = "6200";
public static final String STATEMENT_CLOSED = "statement is closed";
public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!";
public static final String INVALID_VARIABLES = "invalid variables";
public static final String RESULT_SET_IS_CLOSED = "resultSet is closed.";
public static final String RESULT_SET_IS_CLOSED = "resultSet is closed";
public static final String DEFAULT_PORT = "6200";
public static Map<Integer, String> DATATYPE_MAP = null;
public static final long JNI_NULL_POINTER = 0L;
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java 0 → 100644
浏览文件 @ e7a52dfb
package com.taosdata.jdbc;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
public class TSDBError {
private static Map<Integer, String> TSDBErrorMap = new HashMap<>();
static {
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD, "this operation is NOT supported currently!");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variables");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED, "statement is closed");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED, "resultSet is closed");
/**************************************************/
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription");
}
public static String wrapErrMsg(String msg) {
return "TDengine Error: " + msg;
}
public static SQLException createSQLException(int errorNumber) {
// JDBC exception code is less than 0x2350
if (errorNumber <= 0x2350)
return new SQLException(TSDBErrorMap.get(errorNumber));
// JNI exception code is
return new SQLException(wrapErrMsg(TSDBErrorMap.get(errorNumber)));
}
}
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java 0 → 100644
浏览文件 @ e7a52dfb
package com.taosdata.jdbc;
public class TSDBErrorNumbers {
public static final int ERROR_CONNECTION_CLOSED = 0x2301; // connection already closed
public static final int ERROR_UNSUPPORTED_METHOD = 0x2302; //this operation is NOT supported currently!
public static final int ERROR_INVALID_VARIABLE = 0x2303; //invalid variables
public static final int ERROR_STATEMENT_CLOSED = 0x2304; //statement already closed
public static final int ERROR_RESULTSET_CLOSED = 0x2305; //resultSet is closed
public static final int ERROR_SUBSCRIBE_FAILED = 0x2350; //failed to create subscription
private TSDBErrorNumbers() {
}
}
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
浏览文件 @ e7a52dfb
......@@ -14,6 +14,8 @@
*****************************************************************************/
package com.taosdata.jdbc;
import com.taosdata.jdbc.utils.TaosInfo;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.util.List;
......@@ -21,6 +23,8 @@ import java.util.List;
public class TSDBJNIConnector {
private static volatile Boolean isInitialized = false;
private TaosInfo taosInfo = TaosInfo.getInstance();
static {
System.loadLibrary("taos");
System.out.println("java.library.path:" + System.getProperty("java.library.path"));
......@@ -91,7 +95,8 @@ public class TSDBJNIConnector {
*/
public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException {
if (this.taos != TSDBConstants.JNI_NULL_POINTER) {
this.closeConnectionImp(this.taos);
// this.closeConnectionImp(this.taos);
closeConnection();
this.taos = TSDBConstants.JNI_NULL_POINTER;
}
......@@ -99,7 +104,8 @@ public class TSDBJNIConnector {
if (this.taos == TSDBConstants.JNI_NULL_POINTER) {
throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l));
}
// invoke connectImp only here
taosInfo.conn_open_increment();
return true;
}
......@@ -120,6 +126,7 @@ public class TSDBJNIConnector {
Long pSql = 0l;
try {
pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
taosInfo.stmt_count_increment();
} catch (Exception e) {
e.printStackTrace();
this.freeResultSetImp(this.taos, pSql);
......@@ -244,10 +251,11 @@ public class TSDBJNIConnector {
private native int fetchRowImp(long connection, long resultSet, TSDBResultSetRowData rowData);
public int fetchBlock(long resultSet, TSDBResultSetBlockData blockData) {
return this.fetchBlockImp(this.taos, resultSet, blockData);
return this.fetchBlockImp(this.taos, resultSet, blockData);
}
private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData);
/**
* Execute close operation from C to release connection pointer by JNI
*
......@@ -262,6 +270,8 @@ public class TSDBJNIConnector {
} else {
throw new SQLException("Undefined error code returned by TDengine when closing a connection");
}
// invoke closeConnectionImpl only here
taosInfo.connect_close_increment();
}
private native int closeConnectionImp(long connection);
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java
浏览文件 @ e7a52dfb
......@@ -14,13 +14,14 @@
*****************************************************************************/
package com.taosdata.jdbc;
import com.taosdata.jdbc.utils.TaosInfo;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
public class TSDBStatement implements Statement {
private TSDBJNIConnector connector = null;
private TSDBJNIConnector connector;
/**
* To store batched commands
......@@ -67,13 +68,12 @@ public class TSDBStatement implements Statement {
}
public ResultSet executeQuery(String sql) throws SQLException {
if (isClosed) {
throw new SQLException("Invalid method call on a closed statement.");
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
// TODO make sure it is not a update query
pSql = this.connector.executeQuery(sql);
long resultSetPointer = this.connector.getResultSet();
if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
this.connector.freeResultSet(pSql);
......@@ -98,8 +98,8 @@ public class TSDBStatement implements Statement {
}
public int executeUpdate(String sql) throws SQLException {
if (isClosed) {
throw new SQLException("Invalid method call on a closed statement.");
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
// TODO check if current query is update query
......@@ -131,25 +131,33 @@ public class TSDBStatement implements Statement {
}
public int getMaxFieldSize() throws SQLException {
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
return 0;
// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
}
public void setMaxFieldSize(int max) throws SQLException {
if (isClosed())
throw new SQLException(TSDBConstants.STATEMENT_CLOSED);
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
}
public int getMaxRows() throws SQLException {
if (isClosed())
throw new SQLException(TSDBConstants.STATEMENT_CLOSED);
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
// always set maxRows to zero, meaning unlimitted rows in a resultSet
return 0;
}
public void setMaxRows(int max) throws SQLException {
if (isClosed()) {
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
}
// always set maxRows to zero, meaning unlimited rows in a resultSet
}
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java 已删除 100644 → 0
浏览文件 @ 2ee22867
package com.taosdata.jdbc.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.util.*;
import java.util.concurrent.TimeUnit;
public class TDNode {
private int index;
private int running;
private int deployed;
private boolean testCluster;
private String path;
private String cfgDir;
private String dataDir;
private String logDir;
private String cfgPath;
public TDNode(int index) {
this.index = index;
running = 0;
deployed = 0;
testCluster = false;
}
public void setPath(String path) {
this.path = path;
}
public void setTestCluster(boolean testCluster) {
this.testCluster = testCluster;
}
public void setRunning(int running) {
this.running = running;
}
public void searchTaosd(File dir, ArrayList<String> taosdPath) {
File[] fileList = dir.listFiles();
if(fileList == null || fileList.length == 0) {
return;
}
for(File file : fileList) {
if(file.isFile()) {
if(file.getName().equals("taosd")) {
taosdPath.add(file.getAbsolutePath());
}
} else {
searchTaosd(file, taosdPath);
}
}
}
public void start() {
String selfPath = System.getProperty("user.dir");
String binPath = "";
String projDir = selfPath + "/../../../";
try {
ArrayList<String> taosdPath = new ArrayList<>();
File dir = new File(projDir);
String realProjDir = dir.getCanonicalPath();
dir = new File(realProjDir);
System.out.println("project Dir: " + projDir);
searchTaosd(dir, taosdPath);
if(taosdPath.size() == 0) {
System.out.println("The project path doens't exist");
return;
} else {
for(String p : taosdPath) {
if(!p.contains("packaging")) {
binPath = p;
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
if(binPath.isEmpty()) {
System.out.println("taosd not found");
return;
} else {
System.out.println("taosd found in " + binPath);
}
if(this.deployed == 0) {
System.out.println("dnode" + index + "is not deployed");
return;
}
String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & ";
System.out.println("start taosd cmd: " + cmd);
try{
Runtime.getRuntime().exec(cmd);
TimeUnit.SECONDS.sleep(5);
} catch (Exception e) {
e.printStackTrace();
}
this.running = 1;
}
public Integer getTaosdPid() {
String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'";
String[] cmds = {"sh", "-c", cmd};
try {
Process process = Runtime.getRuntime().exec(cmds);
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line = null;
Integer res = null;
while((line = reader.readLine()) != null) {
if(!line.isEmpty()) {
res = Integer.valueOf(line);
break;
}
}
return res;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public void stop() {
if (this.running != 0) {
Integer pid = null;
while((pid = getTaosdPid()) != null) {
String killCmd = "kill -term " + pid;
String[] killCmds = {"sh", "-c", killCmd};
try {
Runtime.getRuntime().exec(killCmds).waitFor();
TimeUnit.SECONDS.sleep(2);
} catch (Exception e) {
e.printStackTrace();
}
}
try {
for(int port = 6030; port < 6041; port ++) {
String fuserCmd = "fuser -k -n tcp " + port;
Runtime.getRuntime().exec(fuserCmd).waitFor();
}
} catch (Exception e) {
e.printStackTrace();
}
this.running = 0;
System.out.println("dnode:" + this.index + " is stopped by kill -term");
}
}
public void startIP() {
try{
String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up";
Runtime.getRuntime().exec(cmd).waitFor();
} catch (Exception e) {
e.printStackTrace();
}
}
public void stopIP() {
try{
String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down";
Runtime.getRuntime().exec(cmd).waitFor();
} catch (Exception e) {
e.printStackTrace();
}
}
public void setCfgConfig(String option, String value) {
try{
String cmd = "echo " + option + " " + value + " >> " + this.cfgPath;
String[] cmdLine = {"sh", "-c", cmd};
Process ps = Runtime.getRuntime().exec(cmdLine);
ps.waitFor();
} catch (Exception e) {
e.printStackTrace();
}
}
public String getDnodeRootDir() {
String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index;
return dnodeRootDir;
}
public String getDnodesRootDir() {
String dnodesRootDir = this.path + "/sim/psim" + this.index;
return dnodesRootDir;
}
public void deploy() {
this.logDir = this.path + "/sim/dnode" + this.index + "/log";
this.dataDir = this.path + "/sim/dnode" + this.index + "/data";
this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg";
this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg";
try {
String cmd = "rm -rf " + this.logDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "rm -rf " + this.cfgDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "rm -rf " + this.dataDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "mkdir -p " + this.logDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "mkdir -p " + this.cfgDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "mkdir -p " + this.dataDir;
Runtime.getRuntime().exec(cmd).waitFor();
cmd = "touch " + this.cfgPath;
Runtime.getRuntime().exec(cmd).waitFor();
} catch (Exception e) {
e.printStackTrace();
}
if(this.testCluster) {
startIP();
setCfgConfig("masterIp", "192.168.0.1");
setCfgConfig("secondIp", "192.168.0.2");
setCfgConfig("publicIp", "192.168.0." + this.index);
setCfgConfig("internalIp", "192.168.0." + this.index);
setCfgConfig("privateIp", "192.168.0." + this.index);
}
setCfgConfig("dataDir", this.dataDir);
setCfgConfig("logDir", this.logDir);
setCfgConfig("numOfLogLines", "1000000/00");
setCfgConfig("mnodeEqualVnodeNum", "0");
setCfgConfig("walLevel", "1");
setCfgConfig("statusInterval", "1");
setCfgConfig("numOfMnodes", "3");
setCfgConfig("numOfThreadsPerCore", "2.0");
setCfgConfig("monitor", "0");
setCfgConfig("maxVnodeConnections", "30000");
setCfgConfig("maxMgmtConnections", "30000");
setCfgConfig("maxMeterConnections", "30000");
setCfgConfig("maxShellConns", "30000");
setCfgConfig("locale", "en_US.UTF-8");
setCfgConfig("charset", "UTF-8");
setCfgConfig("asyncLog", "0");
setCfgConfig("anyIp", "0");
setCfgConfig("dDebugFlag", "135");
setCfgConfig("mDebugFlag", "135");
setCfgConfig("sdbDebugFlag", "135");
setCfgConfig("rpcDebugFlag", "135");
setCfgConfig("tmrDebugFlag", "131");
setCfgConfig("cDebugFlag", "135");
setCfgConfig("httpDebugFlag", "135");
setCfgConfig("monitorDebugFlag", "135");
setCfgConfig("udebugFlag", "135");
setCfgConfig("jnidebugFlag", "135");
setCfgConfig("qdebugFlag", "135");
this.deployed = 1;
}
}
\ No newline at end of file
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java 已删除 100644 → 0
浏览文件 @ 2ee22867
package com.taosdata.jdbc.utils;
import java.io.File;
import java.util.*;
public class TDNodes {
private ArrayList<TDNode> tdNodes;
private boolean testCluster;
public TDNodes () {
tdNodes = new ArrayList<>();
for(int i = 1; i < 11; i ++) {
tdNodes.add(new TDNode(i));
}
}
public void setTestCluster(boolean testCluster) {
this.testCluster = testCluster;
}
public void check(int index) {
if(index < 1 || index > 10) {
System.out.println("index: " + index + " should on a scale of [1, 10]");
return;
}
}
public void deploy(int index) {
try {
File file = new File(System.getProperty("user.dir") + "/../../../");
String projectRealPath = file.getCanonicalPath();
check(index);
tdNodes.get(index - 1).setTestCluster(this.testCluster);
tdNodes.get(index - 1).setPath(projectRealPath);
tdNodes.get(index - 1).deploy();
} catch (Exception e) {
e.printStackTrace();
System.out.println("deploy Test Exception");
}
}
public void cfg(int index, String option, String value) {
check(index);
tdNodes.get(index - 1).setCfgConfig(option, value);
}
public TDNode getTDNode(int index) {
check(index);
return tdNodes.get(index - 1);
}
public void start(int index) {
check(index);
tdNodes.get(index - 1).start();
}
public void stop(int index) {
check(index);
tdNodes.get(index - 1).stop();
}
public void startIP(int index) {
check(index);
tdNodes.get(index - 1).startIP();
}
public void stopIP(int index) {
check(index);
tdNodes.get(index - 1).stopIP();
}
}
\ No newline at end of file
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java 0 → 100644
浏览文件 @ e7a52dfb
package com.taosdata.jdbc.utils;
import javax.management.*;
import java.lang.management.ManagementFactory;
import java.util.concurrent.atomic.AtomicLong;
public class TaosInfo implements TaosInfoMBean {
private static volatile TaosInfo instance;
private AtomicLong connect_open = new AtomicLong();
private AtomicLong connect_close = new AtomicLong();
private AtomicLong statement_count = new AtomicLong();
static {
try {
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo");
server.registerMBean(TaosInfo.getInstance(), name);
} catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) {
e.printStackTrace();
}
}
@Override
public long getConnect_open() {
return connect_open.get();
}
@Override
public long getConnect_close() {
return connect_close.get();
}
@Override
public long getConnect_active() {
return connect_open.get() - connect_close.get();
}
@Override
public long getStatement_count() {
return statement_count.get();
}
/*******************************************************/
public void conn_open_increment() {
connect_open.incrementAndGet();
}
public void connect_close_increment() {
connect_close.incrementAndGet();
}
public void stmt_count_increment() {
statement_count.incrementAndGet();
}
/********************************************************************************/
private TaosInfo() {
}
public static TaosInfo getInstance() {
if (instance == null) {
synchronized (TaosInfo.class) {
if (instance == null) {
instance = new TaosInfo();
}
}
}
return instance;
}
}
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java 0 → 100644
浏览文件 @ e7a52dfb
package com.taosdata.jdbc.utils;
public interface TaosInfoMBean {
long getConnect_open();
long getConnect_close();
long getConnect_active();
long getStatement_count();
}
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java 0 → 100644
浏览文件 @ e7a52dfb
package com.taosdata.jdbc.cases;
import org.junit.Test;
import java.sql.*;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public class TaosInfoMonitorTest {
@Test
public void testCreateTooManyConnection() throws ClassNotFoundException {
Class.forName("com.taosdata.jdbc.TSDBDriver");
final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata";
List<Connection> connectionList = IntStream.range(0, 100).mapToObj(i -> {
try {
TimeUnit.MILLISECONDS.sleep(100);
return DriverManager.getConnection(url);
} catch (SQLException | InterruptedException e) {
e.printStackTrace();
}
return null;
}).collect(Collectors.toList());
connectionList.stream().forEach(conn -> {
try (Statement stmt = conn.createStatement()) {
ResultSet rs = stmt.executeQuery("show databases");
while (rs.next()) {
}
TimeUnit.MILLISECONDS.sleep(100);
} catch (SQLException | InterruptedException e) {
e.printStackTrace();
}
});
connectionList.stream().forEach(conn -> {
try {
conn.close();
TimeUnit.MILLISECONDS.sleep(100);
} catch (SQLException | InterruptedException e) {
e.printStackTrace();
}
});
}
}
src/connector/python/linux/python2/setup.py
浏览文件 @ e7a52dfb
......@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="taos",
version="2.0.4",
version="2.0.5",
author="Taosdata Inc.",
author_email="support@taosdata.com",
description="TDengine python client package",
......
src/connector/python/linux/python2/taos/cursor.py
浏览文件 @ e7a52dfb
from .cinterface import CTaosInterface
from .error import *
from .constants import FieldType
import threading
class TDengineCursor(object):
......@@ -36,7 +35,6 @@ class TDengineCursor(object):
self._block_iter = 0
self._affected_rows = 0
self._logfile = ""
self._threadId = threading.get_ident()
if connection is not None:
self._connection = connection
......
src/cq/src/cqMain.c
浏览文件 @ e7a52dfb
......@@ -91,6 +91,20 @@ void cqRmFromList(SCqObj *pObj) {
}
static void freeSCqContext(void *handle) {
if (handle == NULL) {
return;
}
SCqContext *pContext = handle;
pthread_mutex_destroy(&pContext->mutex);
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
cDebug("vgId:%d, CQ is closed", pContext->vgId);
free(pContext);
}
void cqFree(void *handle) {
if (tsEnableStream == 0) {
return;
......@@ -125,13 +139,7 @@ void cqFree(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
if (delete) {
pthread_mutex_destroy(&pContext->mutex);
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
cDebug("vgId:%d, CQ is closed", pContext->vgId);
free(pContext);
freeSCqContext(pContext);
}
}
......@@ -184,18 +192,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
return pContext;
}
static void freeSCqContext(void *handle) {
if (handle == NULL) {
return;
}
SCqContext *pContext = handle;
pthread_mutex_destroy(&pContext->mutex);
taosTmrCleanUp(pContext->tmrCtrl);
pContext->tmrCtrl = NULL;
cDebug("vgId:%d, CQ is closed", pContext->vgId);
free(pContext);
}
void cqClose(void *handle) {
if (tsEnableStream == 0) {
return;
......@@ -204,7 +201,9 @@ void cqClose(void *handle) {
if (handle == NULL) return;
pContext->delete = 1;
int32_t hasCq = 0;
int32_t existLoop = 0;
// stop all CQs
cqStop(pContext);
......@@ -218,7 +217,13 @@ void cqClose(void *handle) {
cqRmFromList(pObj);
rid = pObj->rid;
} else {
hasCq = 1;
if (pContext->pHead == NULL) {
existLoop = 1;
}
} else {
pthread_mutex_unlock(&pContext->mutex);
break;
}
......@@ -226,9 +231,15 @@ void cqClose(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
taosRemoveRef(cqObjRef, rid);
if (existLoop) {
break;
}
}
freeSCqContext(pContext);
if (hasCq == 0) {
freeSCqContext(pContext);
}
}
void cqStart(void *handle) {
......
src/dnode/src/dnodeShell.c
浏览文件 @ e7a52dfb
......@@ -36,12 +36,12 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
// the following message shall be treated as mnode write
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
......@@ -57,13 +57,13 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
// the following message shall be treated as mnode query
// the following message shall be treated as mnode query
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
......@@ -153,7 +153,7 @@ static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char *
}
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0;
if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0;
int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
if (code != TSDB_CODE_APP_NOT_READY) return code;
......@@ -164,7 +164,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
rpcMsg.pCont = pMsg;
rpcMsg.contLen = sizeof(SAuthMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
dDebug("user:%s, send auth msg to mnodes", user);
SRpcMsg rpcRsp = {0};
dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
......@@ -202,14 +202,14 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
SRpcMsg rpcRsp = {0};
dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
terrno = rpcRsp.code;
if (rpcRsp.code != 0) {
rpcFreeCont(rpcRsp.pCont);
dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid);
return NULL;
} else {
dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid);
// delete this after debug finished
SMDCreateTableMsg *pTable = rpcRsp.pCont;
int16_t numOfColumns = htons(pTable->numOfColumns);
......@@ -231,4 +231,4 @@ SStatisInfo dnodeGetStatisInfo() {
}
return info;
}
\ No newline at end of file
}
src/dnode/src/dnodeStep.c
浏览文件 @ e7a52dfb
......@@ -69,6 +69,6 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) {
return 0;
}
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) {
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) {
taosStepCleanupImp(pSteps, stepSize - 1);
}
\ No newline at end of file
}
src/dnode/src/dnodeSystem.c
浏览文件 @ e7a52dfb
......@@ -181,4 +181,4 @@ static void sigintHandler(int32_t signum, void *sigInfo, void *context) {
#ifdef WINDOWS
tsem_wait(&exitSem);
#endif
}
\ No newline at end of file
}
src/dnode/src/dnodeVWrite.c
浏览文件 @ e7a52dfb
......@@ -53,7 +53,7 @@ void dnodeCleanupVWrite() {
for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(pWorker->qset);
if (pWorker->qset) taosQsetThreadResume(pWorker->qset);
}
}
......
src/kit/taosdemo/taosdemo.c
浏览文件 @ e7a52dfb
......@@ -908,6 +908,7 @@ int main(int argc, char *argv[]) {
}
pthread_join(read_id, NULL);
taos_close(rInfo->taos);
free(rInfo);
}
taos_cleanup();
......
src/kit/taosdemox/taosdemox.c
浏览文件 @ e7a52dfb
......@@ -93,6 +93,8 @@ extern char configDir[];
#define MAX_QUERY_SQL_COUNT 10
#define MAX_QUERY_SQL_LENGTH 256
#define MAX_DATABASE_COUNT 256
typedef enum CREATE_SUB_TALBE_MOD_EN {
PRE_CREATE_SUBTBL,
AUTO_CREATE_SUBTBL,
......@@ -116,7 +118,41 @@ enum QUERY_TYPE {
INSERT_TYPE,
QUERY_TYPE_BUT
} ;
enum _show_db_index {
TSDB_SHOW_DB_NAME_INDEX,
TSDB_SHOW_DB_CREATED_TIME_INDEX,
TSDB_SHOW_DB_NTABLES_INDEX,
TSDB_SHOW_DB_VGROUPS_INDEX,
TSDB_SHOW_DB_REPLICA_INDEX,
TSDB_SHOW_DB_QUORUM_INDEX,
TSDB_SHOW_DB_DAYS_INDEX,
TSDB_SHOW_DB_KEEP_INDEX,
TSDB_SHOW_DB_CACHE_INDEX,
TSDB_SHOW_DB_BLOCKS_INDEX,
TSDB_SHOW_DB_MINROWS_INDEX,
TSDB_SHOW_DB_MAXROWS_INDEX,
TSDB_SHOW_DB_WALLEVEL_INDEX,
TSDB_SHOW_DB_FSYNC_INDEX,
TSDB_SHOW_DB_COMP_INDEX,
TSDB_SHOW_DB_CACHELAST_INDEX,
TSDB_SHOW_DB_PRECISION_INDEX,
TSDB_SHOW_DB_UPDATE_INDEX,
TSDB_SHOW_DB_STATUS_INDEX,
TSDB_MAX_SHOW_DB
};
// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
TSDB_SHOW_STABLES_NAME_INDEX,
TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
TSDB_SHOW_STABLES_COLUMNS_INDEX,
TSDB_SHOW_STABLES_METRIC_INDEX,
TSDB_SHOW_STABLES_UID_INDEX,
TSDB_SHOW_STABLES_TID_INDEX,
TSDB_SHOW_STABLES_VGID_INDEX,
TSDB_MAX_SHOW_STABLES
};
enum _describe_table_index {
TSDB_DESCRIBE_METRIC_FIELD_INDEX,
TSDB_DESCRIBE_METRIC_TYPE_INDEX,
......@@ -173,6 +209,7 @@ typedef struct SSuperTable_S {
int childTblCount;
bool superTblExists; // 0: no, 1: yes
bool childTblExists; // 0: no, 1: yes
int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql
int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table
char childTblPrefix[MAX_TB_NAME_SIZE];
char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample
......@@ -218,6 +255,28 @@ typedef struct SSuperTable_S {
int64_t totalAffectedRows;
} SSuperTable;
typedef struct {
char name[TSDB_DB_NAME_LEN + 1];
char create_time[32];
int32_t ntables;
int32_t vgroups;
int16_t replica;
int16_t quorum;
int16_t days;
char keeplist[32];
int32_t cache; //MB
int32_t blocks;
int32_t minrows;
int32_t maxrows;
int8_t wallevel;
int32_t fsync;
int8_t comp;
int8_t cachelast;
char precision[8]; // time resolution
int8_t update;
char status[16];
} SDbInfo;
typedef struct SDbCfg_S {
// int maxtablesPerVnode;
int minRows;
......@@ -808,13 +867,14 @@ static void init_rand_data() {
static void printfInsertMeta() {
printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n");
printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
printf("user: \033[33m%s\033[0m\n", g_Dbs.user);
printf("password: \033[33m%s\033[0m\n", g_Dbs.password);
printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile);
printf("thread count: \033[33m%d\033[0m\n", g_Dbs.threadCount);
printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount);
printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
printf("user: \033[33m%s\033[0m\n", g_Dbs.user);
printf("password: \033[33m%s\033[0m\n", g_Dbs.password);
printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile);
printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount);
printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount);
for (int i = 0; i < g_Dbs.dbCount; i++) {
printf("database[\033[33m%d\033[0m]:\n", i);
printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName);
......@@ -944,11 +1004,12 @@ static void printfInsertMeta() {
static void printfInsertMetaToFile(FILE* fp) {
fprintf(fp, "================ insert.json parse result START================\n");
fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port);
fprintf(fp, "user: %s\n", g_Dbs.user);
fprintf(fp, "password: %s\n", g_Dbs.password);
fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile);
fprintf(fp, "thread count: %d\n", g_Dbs.threadCount);
fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port);
fprintf(fp, "user: %s\n", g_Dbs.user);
fprintf(fp, "password: %s\n", g_Dbs.password);
fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile);
fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount);
fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
fprintf(fp, "database count: %d\n", g_Dbs.dbCount);
for (int i = 0; i < g_Dbs.dbCount; i++) {
......@@ -1123,6 +1184,272 @@ static void printfQueryMeta() {
printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n");
}
static char* xFormatTimestamp(char* buf, int64_t val, int precision) {
time_t tt;
if (precision == TSDB_TIME_PRECISION_MICRO) {
tt = (time_t)(val / 1000000);
} else {
tt = (time_t)(val / 1000);
}
/* comment out as it make testcases like select_with_tags.sim fail.
but in windows, this may cause the call to localtime crash if tt < 0,
need to find a better solution.
if (tt < 0) {
tt = 0;
}
*/
#ifdef WINDOWS
if (tt < 0) tt = 0;
#endif
struct tm* ptm = localtime(&tt);
size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);
if (precision == TSDB_TIME_PRECISION_MICRO) {
sprintf(buf + pos, ".%06d", (int)(val % 1000000));
} else {
sprintf(buf + pos, ".%03d", (int)(val % 1000));
}
return buf;
}
static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) {
if (val == NULL) {
fprintf(fp, "%s", TSDB_DATA_NULL_STR);
return;
}
char buf[TSDB_MAX_BYTES_PER_ROW];
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
break;
case TSDB_DATA_TYPE_TINYINT:
fprintf(fp, "%d", *((int8_t *)val));
break;
case TSDB_DATA_TYPE_SMALLINT:
fprintf(fp, "%d", *((int16_t *)val));
break;
case TSDB_DATA_TYPE_INT:
fprintf(fp, "%d", *((int32_t *)val));
break;
case TSDB_DATA_TYPE_BIGINT:
fprintf(fp, "%" PRId64, *((int64_t *)val));
break;
case TSDB_DATA_TYPE_FLOAT:
fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
break;
case TSDB_DATA_TYPE_DOUBLE:
fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
memcpy(buf, val, length);
buf[length] = 0;
fprintf(fp, "\'%s\'", buf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
xFormatTimestamp(buf, *(int64_t*)val, precision);
fprintf(fp, "'%s'", buf);
break;
default:
break;
}
}
static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
}
FILE* fp = fopen(fname, "at");
if (fp == NULL) {
fprintf(stderr, "ERROR: failed to open file: %s\n", fname);
return -1;
}
int num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int precision = taos_result_precision(tres);
for (int col = 0; col < num_fields; col++) {
if (col > 0) {
fprintf(fp, ",");
}
fprintf(fp, "%s", fields[col].name);
}
fputc('\n', fp);
int numOfRows = 0;
do {
int32_t* length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
if (i > 0) {
fputc(',', fp);
}
xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
}
fputc('\n', fp);
numOfRows++;
row = taos_fetch_row(tres);
} while( row != NULL);
fclose(fp);
return numOfRows;
}
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
TAOS_RES * res;
TAOS_ROW row = NULL;
int count = 0;
res = taos_query(taos, "show databases;");
int32_t code = taos_errno(res);
if (code != 0) {
fprintf(stderr, "failed to run <show databases>, reason: %s\n", taos_errstr(res));
return -1;
}
TAOS_FIELD *fields = taos_fetch_fields(res);
while ((row = taos_fetch_row(res)) != NULL) {
// sys database name : 'log'
if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue;
dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
if (dbInfos[count] == NULL) {
fprintf(stderr, "failed to allocate memory for some dbInfo[%d]\n", count);
return -1;
}
strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
xFormatTimestamp(dbInfos[count]->create_time, *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], TSDB_TIME_PRECISION_MILLI);
dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
strncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
count++;
if (count > MAX_DATABASE_COUNT) {
fprintf(stderr, "The database count overflow than %d\n", MAX_DATABASE_COUNT);
break;
}
}
return count;
}
static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) {
FILE *fp = NULL;
if (filename[0] != 0) {
fp = fopen(filename, "at");
if (fp == NULL) {
fprintf(stderr, "failed to open file: %s\n", filename);
return;
}
}
fprintf(fp, "================ database[%d] ================\n", index);
fprintf(fp, "name: %s\n", dbInfos->name);
fprintf(fp, "created_time: %s\n", dbInfos->create_time);
fprintf(fp, "ntables: %d\n", dbInfos->ntables);
fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
fprintf(fp, "replica: %d\n", dbInfos->replica);
fprintf(fp, "quorum: %d\n", dbInfos->quorum);
fprintf(fp, "days: %d\n", dbInfos->days);
fprintf(fp, "keep1,keep2,keep(D): %s\n", dbInfos->keeplist);
fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
fprintf(fp, "blocks: %d\n", dbInfos->blocks);
fprintf(fp, "minrows: %d\n", dbInfos->minrows);
fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
fprintf(fp, "fsync: %d\n", dbInfos->fsync);
fprintf(fp, "comp: %d\n", dbInfos->comp);
fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
fprintf(fp, "precision: %s\n", dbInfos->precision);
fprintf(fp, "update: %d\n", dbInfos->update);
fprintf(fp, "status: %s\n", dbInfos->status);
fprintf(fp, "\n");
fclose(fp);
}
static void printfQuerySystemInfo(TAOS * taos) {
char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
TAOS_RES* res;
time_t t;
struct tm* lt;
time(&t);
lt = localtime(&t);
snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec);
// show variables
res = taos_query(taos, "show variables;");
//getResult(res, filename);
xDumpResultToFile(filename, res);
// show dnodes
res = taos_query(taos, "show dnodes;");
xDumpResultToFile(filename, res);
//getResult(res, filename);
// show databases
res = taos_query(taos, "show databases;");
SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
if (dbInfos == NULL) {
fprintf(stderr, "failed to allocate memory\n");
return;
}
int dbCount = getDbFromServer(taos, dbInfos);
if (dbCount <= 0) return;
for (int i = 0; i < dbCount; i++) {
// printf database info
printfDbInfoForQueryToFile(filename, dbInfos[i], i);
// show db.vgroups
snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
res = taos_query(taos, buffer);
xDumpResultToFile(filename, res);
// show db.stables
snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
res = taos_query(taos, buffer);
xDumpResultToFile(filename, res);
free(dbInfos[i]);
}
free(dbInfos);
}
#ifdef TD_LOWA_CURL
static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
......@@ -1730,19 +2057,27 @@ static int createDatabases() {
void * createTable(void *sarg)
{
char command[BUFFER_SIZE] = "\0";
{
threadInfo *winfo = (threadInfo *)sarg;
SSuperTable* superTblInfo = winfo->superTblInfo;
int64_t lastPrintTime = taosGetTimestampMs();
char* buffer = calloc(superTblInfo->maxSqlLen, 1);
int len = 0;
int batchNum = 0;
//printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) {
if (0 == g_Dbs.use_metric) {
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
snprintf(buffer, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
} else {
if (0 == len) {
batchNum = 0;
memset(buffer, 0, superTblInfo->maxSqlLen);
len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "create table ");
}
char* tagsValBuf = NULL;
if (0 == superTblInfo->tagSource) {
tagsValBuf = generateTagVaulesForStb(superTblInfo);
......@@ -1750,13 +2085,22 @@ void * createTable(void *sarg)
tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount);
}
if (NULL == tagsValBuf) {
free(buffer);
return NULL;
}
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
free(tagsValBuf);
batchNum++;
if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) {
continue;
}
}
if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){
len = 0;
if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){
free(buffer);
return NULL;
}
......@@ -1766,7 +2110,12 @@ void * createTable(void *sarg)
lastPrintTime = currentPrintTime;
}
}
if (0 != len) {
(void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE);
}
free(buffer);
return NULL;
}
......@@ -2422,6 +2771,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
printf("failed to read json, auto_create_table not found");
goto PARSE_OVER;
}
cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
} else if (!batchCreateTbl) {
g_Dbs.db[i].superTbls[j].batchCreateTableNum = 2000;
} else {
printf("failed to read json, batch_create_tbl_num not found");
goto PARSE_OVER;
}
cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) {
......@@ -3679,14 +4038,14 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
b = ntables % threads;
}
TAOS* taos;
if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
if (NULL == taos) {
printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
exit(-1);
}
}
//TAOS* taos;
//if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
// taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
// if (NULL == taos) {
// printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
// exit(-1);
// }
//}
int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
if (0 != precision[0]) {
......@@ -3719,7 +4078,12 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
t_info->start_time = start_time;
if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
t_info->taos = taos;
//t_info->taos = taos;
t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
if (NULL == t_info->taos) {
printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL));
exit(-1);
}
} else {
t_info->taos = NULL;
#ifdef TD_LOWA_CURL
......@@ -3754,6 +4118,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
threadInfo *t_info = infos + i;
tsem_destroy(&(t_info->lock_sem));
taos_close(t_info->taos);
superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
superTblInfo->totalRowsInserted += t_info->totalRowsInserted;
......@@ -3766,7 +4131,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
double end = getCurrentTime();
taos_close(taos);
//taos_close(taos);
free(pids);
free(infos);
......@@ -4093,7 +4458,7 @@ void *subQueryProcess(void *sarg) {
int queryTestProcess() {
TAOS * taos = NULL;
taos_init();
taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port);
taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, NULL, g_queryInfo.port);
if (taos == NULL) {
fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
exit(-1);
......@@ -4106,6 +4471,8 @@ int queryTestProcess() {
printfQueryMeta();
printf("Press enter key to continue\n\n");
(void)getchar();
printfQuerySystemInfo(taos);
pthread_t *pids = NULL;
threadInfo *infos = NULL;
......
src/mnode/src/mnodeDb.c
浏览文件 @ e7a52dfb
......@@ -679,7 +679,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb);
if (pDb == NULL) break;
if (pDb->pAcct != pUser->pAcct) {
if (pDb->pAcct != pUser->pAcct || pDb->status != TSDB_DB_STATUS_READY) {
mnodeDecDbRef(pDb);
continue;
}
......
src/mnode/src/mnodeTable.c
浏览文件 @ e7a52dfb
......@@ -1081,20 +1081,13 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
if (code != TSDB_CODE_SUCCESS) {
mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
} else {
mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
return code;
}
return code;
}
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash));
if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
while (pVgId) {
......@@ -1122,6 +1115,16 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
mnodeDropAllChildTablesInStable(pStable);
}
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash));
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsSuperTableSdb,
......
src/os/inc/osDarwin.h
浏览文件 @ e7a52dfb
......@@ -85,6 +85,7 @@ extern "C" {
#define TAOS_OS_FUNC_STRING_STR2INT64
#define TAOS_OS_FUNC_SYSINFO
#define TAOS_OS_FUNC_TIMER
#define TAOS_OS_FUNC_SEMPHONE_PTHREAD
// specific
#define htobe64 htonll
......@@ -114,7 +115,7 @@ int64_t tsosStr2int64(char *str);
void taos_block_sigalrm(void);
#define TAOS_OS_DEF_EPOLL
#define TAOS_EPOLL_WAIT_TIME 500
#define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET;
typedef SOCKET EpollFd;
#define EpollClose(pollFd) epoll_close(pollFd)
......
src/os/inc/osDef.h
浏览文件 @ e7a52dfb
......@@ -26,10 +26,6 @@ extern "C" {
#endif
#endif
#ifndef STDERR_FILENO
#define STDERR_FILENO (2)
#endif
#define FD_VALID(x) ((x) > STDERR_FILENO)
#define FD_INITIALIZER ((int32_t)-1)
......
src/os/inc/osWindows.h
浏览文件 @ e7a52dfb
......@@ -46,6 +46,8 @@
#include "msvcFcntl.h"
#include "msvcLibgen.h"
#include "msvcStdio.h"
#include "msvcUnistd.h"
#include "msvcLibgen.h"
#include "sys/msvcStat.h"
#include "sys/msvcTypes.h"
......@@ -144,7 +146,6 @@ typedef int (*__compar_fn_t)(const void *, const void *);
#define in_addr_t unsigned long
#define socklen_t int
#define htobe64 htonll
#define getpid _getpid
struct tm *localtime_r(const time_t *timep, struct tm *result);
char * strptime(const char *buf, const char *fmt, struct tm *tm);
......@@ -153,15 +154,8 @@ char * getpass(const char *prefix);
int flock(int fd, int option);
int fsync(int filedes);
char * strndup(const char *s, size_t n);
char * dirname(char *pszPathname);
int gettimeofday(struct timeval *ptv, void *pTimeZone);
// for access function in io.h
#define F_OK 00 //Existence only
#define W_OK 02 //Write - only
#define R_OK 04 //Read - only
#define X_OK 06 //Read and write
// for send function in tsocket.c
#define MSG_NOSIGNAL 0
#define SO_NO_CHECK 0x1234
......@@ -208,8 +202,6 @@ typedef struct {
int wordexp(char *words, wordexp_t *pwordexp, int flags);
void wordfree(wordexp_t *pwordexp);
char *realpath(char *path, char *resolved_path);
#define openlog(a, b, c)
#define closelog()
#define LOG_ERR 0
......
src/os/src/darwin/darwinEnv.c
浏览文件 @ e7a52dfb
......@@ -17,17 +17,29 @@
#include "os.h"
#include "tglobal.h"
static const char* expand_like_shell(const char *path) {
static __thread char buf[TSDB_FILENAME_LEN];
buf[0] = '\0';
wordexp_t we;
if (wordexp(path, &we, 0)) return "/tmp/taosd";
if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd";
wordfree(&we);
return buf;
}
void osInit() {
if (configDir[0] == 0) {
strcpy(configDir, "~/TDengine/cfg");
strcpy(configDir, expand_like_shell("~/TDengine/cfg"));
}
strcpy(tsVnodeDir, "");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(tsDataDir, "~/TDengine/data");
strcpy(tsLogDir, "~/TDengine/log");
strcpy(tsScriptDir, "~/TDengine/cfg");
strcpy(tsDataDir, expand_like_shell("~/TDengine/data"));
strcpy(tsLogDir, expand_like_shell("~/TDengine/log"));
strcpy(tsScriptDir, expand_like_shell("~/TDengine/cfg"));
strcpy(tsOsName, "Darwin");
}
src/os/src/darwin/darwinFile.c
浏览文件 @ e7a52dfb
......@@ -17,71 +17,49 @@
#include "os.h"
#include "tulog.h"
#define _SEND_FILE_STEP_ 1000
int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
int writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
} else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
return (int)(writeLen + rlen);
} else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
writeLen += _SEND_FILE_STEP_;
}
int r = 0;
if (offset) {
r = fseek(in_file, *offset, SEEK_SET);
if (r==-1) return -1;
}
int remain = count - writeLen;
if (remain > 0) {
size_t rlen = fread(buffer, 1, remain, in_file);
if (rlen <= 0) {
return writeLen;
} else {
fwrite(buffer, 1, remain, out_file);
writeLen += remain;
off_t len = count;
while (len>0) {
char buf[1024*16];
off_t n = sizeof(buf);
if (len<n) n = len;
size_t m = fread(buf, 1, n, in_file);
if (m<n) {
int e = ferror(in_file);
if (e) return -1;
}
if (m==0) break;
if (m!=fwrite(buf, 1, m, out_file)) {
return -1;
}
len -= m;
}
return writeLen;
return count - len;
}
int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) {
lseek(sfd, (int32_t)(*offset), 0);
int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
if (rlen <= 0) {
return writeLen;
}
else if (rlen < _SEND_FILE_STEP_) {
taosWriteSocket(dfd, buffer, rlen);
return (int64_t)(writeLen + rlen);
}
else {
taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
writeLen += _SEND_FILE_STEP_;
}
int r = 0;
if (offset) {
r = lseek(sfd, *offset, SEEK_SET);
if (r==-1) return -1;
}
int64_t remain = count - writeLen;
if (remain > 0) {
int32_t rlen = read(sfd, buffer, (int32_t)remain);
if (rlen <= 0) {
return writeLen;
}
else {
taosWriteSocket(sfd, buffer, (int32_t)remain);
writeLen += remain;
}
off_t len = count;
while (len>0) {
char buf[1024*16];
off_t n = sizeof(buf);
if (len<n) n = len;
size_t m = read(sfd, buf, n);
if (m==-1) return -1;
if (m==0) break;
size_t l = write(dfd, buf, m);
if (l==-1) return -1;
len -= l;
}
return writeLen;
return count - len;
}
src/os/src/darwin/darwinSemphone.c
浏览文件 @ e7a52dfb
......@@ -21,6 +21,8 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include <libproc.h>
// #define SEM_USE_PTHREAD
// #define SEM_USE_POSIX
#define SEM_USE_SEM
......@@ -279,3 +281,41 @@ int tsem_destroy(tsem_t *sem) {
return 0;
}
bool taosCheckPthreadValid(pthread_t thread) {
uint64_t id = 0;
int r = pthread_threadid_np(thread, &id);
return r ? false : true;
}
int64_t taosGetSelfPthreadId() {
return (int64_t)pthread_self();
}
int64_t taosGetPthreadId(pthread_t thread) {
return (int64_t)thread;
}
void taosResetPthread(pthread_t* thread) {
*thread = NULL;
}
bool taosComparePthread(pthread_t first, pthread_t second) {
return pthread_equal(first, second) ? true : false;
}
int32_t taosGetPId() {
return (int32_t)getpid();
}
int32_t taosGetCurrentAPPName(char* name, int32_t* len) {
char buf[PATH_MAX+1];
buf[0] = '\0';
proc_name(getpid(), buf, sizeof(buf)-1);
buf[PATH_MAX] = '\0';
size_t n = strlen(buf);
if (len) *len = n;
if (name) strcpy(name, buf);
return 0;
}
src/os/src/darwin/darwinSysInfo.c
浏览文件 @ e7a52dfb
......@@ -24,42 +24,134 @@
static void taosGetSystemTimezone() {
// get and set default timezone
SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *tz = getenv("TZ");
if (tz == NULL || strlen(tz) == 0) {
strcpy(tsTimezone, "not configured");
if (cfg_timezone == NULL) return;
if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) {
return;
}
/* load time zone string from /etc/localtime */
char buf[4096];
char *tz = NULL; {
int n = readlink("/etc/localtime", buf, sizeof(buf));
if (n<0) {
uError("read /etc/localtime error, reason:%s", strerror(errno));
return;
}
buf[n] = '\0';
for(int i=n-1; i>=0; --i) {
if (buf[i]=='/') {
if (tz) {
tz = buf + i + 1;
break;
}
tz = buf + i + 1;
}
}
else {
strcpy(tsTimezone, tz);
if (!tz || 0==strchr(tz, '/')) {
uError("parsing /etc/localtime failed");
return;
}
cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uInfo("timezone not configured, use default");
setenv("TZ", tz, 1);
tzset();
}
/*
* NOTE: do not remove it.
* Enforce set the correct daylight saving time(DST) flag according
* to current time
*/
time_t tx1 = time(NULL);
struct tm tm1;
localtime_r(&tx1, &tm1);
/*
* format example:
*
* Asia/Shanghai (CST, +0800)
* Europe/London (BST, +0100)
*/
snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)",
tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600);
// cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uWarn("timezone not configured, set to system default:%s", tsTimezone);
}
static void taosGetSystemLocale() {
// get and set default locale
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
static void taosGetSystemLocale() { // get and set default locale
char sep = '.';
char *locale = NULL;
SGlobalCfg *cfg_locale = taosGetConfigOption("locale");
if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uInfo("locale not configured, set to default:%s", tsLocale);
uWarn("locale not configured, set to system default:%s", tsLocale);
}
}
/* if user does not specify the charset, extract it from locale */
SGlobalCfg *cfg_charset = taosGetConfigOption("charset");
if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
strcpy(tsCharset, "cp936");
cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
uInfo("charset not configured, set to default:%s", tsCharset);
char *str = strrchr(tsLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
free(revisedCharset);
uWarn("charset not configured, set to system default:%s", tsCharset);
} else {
strcpy(tsCharset, "UTF-8");
uWarn("can't get locale and charset from system, set it to UTF-8");
}
}
}
void taosPrintOsInfo() {}
void taosPrintOsInfo() {
uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize / 1024);
// uInfo(" os openMax: %" PRId64, tsOpenMax);
// uInfo(" os streamMax: %" PRId64, tsStreamMax);
uInfo(" os numOfCores: %d", tsNumOfCores);
uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB);
uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB);
struct utsname buf;
if (uname(&buf)) {
uInfo(" can't fetch os info");
return;
}
uInfo(" os sysname: %s", buf.sysname);
uInfo(" os nodename: %s", buf.nodename);
uInfo(" os release: %s", buf.release);
uInfo(" os version: %s", buf.version);
uInfo(" os machine: %s", buf.machine);
uInfo("==================================");
}
void taosKillSystem() {
uError("function taosKillSystem, exit!");
......@@ -67,6 +159,22 @@ void taosKillSystem() {
}
void taosGetSystemInfo() {
// taosGetProcInfos();
tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN);
long physical_pages = sysconf(_SC_PHYS_PAGES);
long page_size = sysconf(_SC_PAGESIZE);
tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024);
tsPageSize = page_size;
// float tmp1, tmp2;
// taosGetSysMemory(&tmp1);
// taosGetProcMemory(&tmp2);
// taosGetDisk();
// taosGetBandSpeed(&tmp1);
// taosGetCpuUsage(&tmp1, &tmp2);
// taosGetProcIO(&tmp1, &tmp2);
taosGetSystemTimezone();
taosGetSystemLocale();
}
......@@ -121,7 +229,6 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
char cmdline[1024];
char *taosGetCmdlineByPID(int pid) {
errno = 0;
if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) {
......@@ -136,6 +243,7 @@ bool taosGetSystemUid(char *uid) {
uuid_t uuid = {0};
uuid_generate(uuid);
// it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null
uuid_unparse(uuid, uid);
uuid_unparse_lower(uuid, uid);
return true;
}
src/os/src/detail/osFile.c
浏览文件 @ e7a52dfb
......@@ -142,6 +142,8 @@ int64_t taosCopy(char *from, char *to) {
if (bytes < sizeof(buffer)) break;
}
fsync(fidto);
close(fidfrom);
close(fidto);
return size;
......
src/os/src/detail/osSemphone.c
浏览文件 @ e7a52dfb
......@@ -62,4 +62,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) {
return 0;
}
#endif
\ No newline at end of file
#endif
src/os/src/windows/wString.c
浏览文件 @ e7a52dfb
......@@ -75,18 +75,6 @@ char *getpass(const char *prefix) {
return passwd;
}
char *strndup(const char *s, size_t n) {
size_t len = strlen(s);
if (len >= n) {
len = n;
}
char *r = calloc(len + 1, 1);
memcpy(r, s, len);
r[len] = 0;
return r;
}
int twcslen(const wchar_t *wcs) {
int *wstr = (int *)wcs;
if (NULL == wstr) {
......
src/os/src/windows/wWordexp.c
浏览文件 @ e7a52dfb
......@@ -38,7 +38,3 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) {
}
void wordfree(wordexp_t *pwordexp) {}
char *realpath(char *path, char *resolved_path) {
return _fullpath(path, resolved_path, TSDB_FILENAME_LEN - 1);
}
\ No newline at end of file
src/plugins/http/src/httpSystem.c
浏览文件 @ e7a52dfb
......@@ -92,6 +92,11 @@ void httpStopSystem() {
tsHttpServer.stop = 1;
#ifdef WINDOWS
closesocket(tsHttpServer.fd);
#elif __APPLE__
if (tsHttpServer.fd!=-1) {
close(tsHttpServer.fd);
tsHttpServer.fd = -1;
}
#else
shutdown(tsHttpServer.fd, SHUT_RD);
#endif
......
src/query/src/qParserImpl.c
浏览文件 @ e7a52dfb
......@@ -279,7 +279,7 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
pExpr->nSQLOptr = optrType;
pExpr->pLeft = pLeft;
if (pRight == NULL) {
if (pLeft != NULL && pRight == NULL) {
pRight = calloc(1, sizeof(tSQLExpr));
}
......
src/rpc/src/rpcTcp.c
浏览文件 @ e7a52dfb
......@@ -17,7 +17,7 @@
#include "tsocket.h"
#include "tutil.h"
#include "taosdef.h"
#include "taoserror.h"
#include "taoserror.h"
#include "rpcLog.h"
#include "rpcHead.h"
#include "rpcTcp.h"
......@@ -81,7 +81,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
if (pServerObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
......@@ -95,7 +95,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads);
if (pServerObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
free(pServerObj);
return NULL;
}
......@@ -105,18 +105,18 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
// initialize parameters in case it may encounter error later
// initialize parameters in case it may encounter error later
for (int i = 0; i < numOfThreads; ++i) {
pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1);
if (pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
for (int j=0; j<i; ++j) free(pServerObj->pThreadObj[j]);
free(pServerObj->pThreadObj);
free(pServerObj);
return NULL;
}
pServerObj->pThreadObj[i] = pThreadObj;
pThreadObj->pollFd = -1;
taosResetPthread(&pThreadObj->thread);
......@@ -152,9 +152,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
}
pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->fd < 0) code = -1;
if (pServerObj->fd < 0) code = -1;
if (code == 0) {
if (code == 0) {
code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj);
if (code != 0) {
tError("%s failed to create TCP accept thread(%s)", label, strerror(code));
......@@ -162,7 +162,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
}
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
taosCleanUpTcpServer(pServerObj);
pServerObj = NULL;
} else {
......@@ -175,8 +175,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
static void taosStopTcpThread(SThreadObj* pThreadObj) {
if (pThreadObj == NULL) { return;}
// save thread into local variable and signal thread to stop
pthread_t thread = pThreadObj->thread;
// save thread into local variable and signal thread to stop
pthread_t thread = pThreadObj->thread;
if (!taosCheckPthreadValid(thread)) {
return;
}
......@@ -197,6 +197,11 @@ void taosStopTcpServer(void *handle) {
if (pServerObj->fd >= 0) {
#ifdef WINDOWS
closesocket(pServerObj->fd);
#elif defined(__APPLE__)
if (pServerObj->fd!=-1) {
close(pServerObj->fd);
pServerObj->fd = -1;
}
#else
shutdown(pServerObj->fd, SHUT_RD);
#endif
......@@ -265,7 +270,7 @@ static void *taosAcceptTcpConnection(void *arg) {
taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
continue;
}
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj[threadId];
......@@ -274,13 +279,13 @@ static void *taosAcceptTcpConnection(void *arg) {
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
pFdObj->port = htons(caddr.sin_port);
tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
} else {
taosCloseSocket(connFd);
tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
}
}
// pick up next thread for next connection
threadId++;
......@@ -295,17 +300,17 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj));
if (pClientObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
}
tstrncpy(pClientObj->label, label, sizeof(pClientObj->label));
pClientObj->numOfThreads = numOfThreads;
pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*));
pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*));
if (pClientObj->pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
tfree(pClientObj);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
}
int code = 0;
......@@ -317,7 +322,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj));
if (pThreadObj == NULL) {
tError("TCP:%s no enough memory", label);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
for (int j=0; j<i; ++j) free(pClientObj->pThreadObj[j]);
free(pClientObj);
pthread_attr_destroy(&thattr);
......@@ -356,11 +361,11 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
pThreadObj->threadId = i;
}
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
taosCleanUpTcpClient(pClientObj);
pClientObj = NULL;
}
return pClientObj;
}
return pClientObj;
}
void taosStopTcpClient(void *chandle) {
......@@ -374,20 +379,20 @@ void taosStopTcpClient(void *chandle) {
void taosCleanUpTcpClient(void *chandle) {
SClientObj *pClientObj = chandle;
if (pClientObj == NULL) return;
for (int i = 0; i < pClientObj->numOfThreads; ++i) {
for (int i = 0; i < pClientObj->numOfThreads; ++i) {
SThreadObj *pThreadObj= pClientObj->pThreadObj[i];
taosStopTcpThread(pThreadObj);
taosStopTcpThread(pThreadObj);
}
tDebug("%s TCP client is cleaned up", pClientObj->label);
tfree(pClientObj->pThreadObj);
tfree(pClientObj);
tfree(pClientObj);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SClientObj * pClientObj = shandle;
int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads;
atomic_store_32(&pClientObj->index, index + 1);
atomic_store_32(&pClientObj->index, index + 1);
SThreadObj *pThreadObj = pClientObj->pThreadObj[index];
SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
......@@ -402,12 +407,12 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
}
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = ip;
tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
} else {
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
......@@ -422,7 +427,7 @@ void taosCloseTcpConnection(void *chandle) {
if (pFdObj == NULL || pFdObj->signature != pFdObj) return;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj);
tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj);
// pFdObj->thandle = NULL;
pFdObj->closedByApp = 1;
......@@ -435,7 +440,7 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
SThreadObj *pThreadObj = pFdObj->pThreadObj;
int ret = taosWriteMsg(pFdObj->fd, data, len);
tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret);
tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret);
return ret;
}
......@@ -458,7 +463,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
recvInfo.chandle = NULL;
recvInfo.connType = RPC_CONN_TCP;
(*(pThreadObj->processData))(&recvInfo);
}
}
taosFreeFdObj(pFdObj);
}
......@@ -473,7 +478,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
return -1;
return -1;
}
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
......@@ -491,14 +496,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s %p read error, leftLen:%d retLen:%d FD:%p",
tError("%s %p read error, leftLen:%d retLen:%d FD:%p",
pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
free(buffer);
return -1;
}
memcpy(msg, &rpcHead, sizeof(SRpcHead));
pInfo->msg = msg;
pInfo->msgLen = msgLen;
pInfo->ip = pFdObj->ip;
......@@ -509,7 +514,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
pInfo->connType = RPC_CONN_TCP;
if (pFdObj->closedByApp) {
free(buffer);
free(buffer);
return -1;
}
......@@ -557,7 +562,7 @@ static void *taosProcessTcpData(void *param) {
}
if (taosReadTcpData(pFdObj, &recvInfo) < 0) {
shutdown(pFdObj->fd, SHUT_WR);
shutdown(pFdObj->fd, SHUT_WR);
continue;
}
......@@ -565,7 +570,7 @@ static void *taosProcessTcpData(void *param) {
if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj);
}
if (pThreadObj->stop) break;
if (pThreadObj->stop) break;
}
if (pThreadObj->pollFd >=0) {
......@@ -603,7 +608,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
tfree(pFdObj);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
......@@ -637,7 +642,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
tError("%s %p TCP thread:%d, number of FDs is negative!!!",
tError("%s %p TCP thread:%d, number of FDs is negative!!!",
pThreadObj->label, pFdObj->thandle, pThreadObj->threadId);
if (pFdObj->prev) {
......@@ -652,7 +657,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d",
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds);
tfree(pFdObj);
......
src/sync/src/syncTcp.c
浏览文件 @ e7a52dfb
......@@ -103,6 +103,11 @@ void syncCloseTcpThreadPool(void *param) {
#ifdef WINDOWS
closesocket(pPool->acceptFd);
#elif defined(__APPLE__)
if (pPool->acceptFd!=-1) {
close(pPool->acceptFd);
pPool->acceptFd = -1;
}
#else
shutdown(pPool->acceptFd, SHUT_RD);
#endif
......
src/tfs/src/tfs.c
浏览文件 @ e7a52dfb
......@@ -593,7 +593,7 @@ void taosGetDisk() {
tsAvailLogDirGB = (float)(diskSize.avail / unit);
}
if (taosGetDiskSize("/tmp", &diskSize) == 0) {
if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
}
......
src/tsdb/inc/tsdbFile.h
浏览文件 @ e7a52dfb
......@@ -20,6 +20,8 @@
#define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
#define TSDB_IVLD_FID INT_MIN
#define TSDB_FILE_STATE_OK 0
#define TSDB_FILE_STATE_BAD 1
#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f))
......@@ -31,6 +33,10 @@
#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
......@@ -47,10 +53,11 @@ typedef struct {
SMFInfo info;
TFILE f;
int fd;
uint8_t state;
} SMFile;
void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
void tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile);
void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
int tsdbEncodeSMFile(void** buf, SMFile* pMFile);
void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
......@@ -165,6 +172,7 @@ typedef struct {
SDFInfo info;
TFILE f;
int fd;
uint8_t state;
} SDFile;
void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
......@@ -346,4 +354,14 @@ static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid,
*maxKey = *minKey + days * tsMsPerDay[precision] - 1;
}
static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
return false;
}
}
return true;
}
#endif /* _TS_TSDB_FILE_H_ */
\ No newline at end of file
src/tsdb/src/tsdbCommit.c
浏览文件 @ e7a52dfb
......@@ -52,7 +52,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo);
static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen);
static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid);
static int tsdbCommitTSData(STsdbRepo *pRepo);
static int tsdbStartCommit(STsdbRepo *pRepo);
static void tsdbStartCommit(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static int tsdbCreateCommitIters(SCommitH *pCommith);
......@@ -84,10 +84,7 @@ static int tsdbApplyRtn(STsdbRepo *pRepo);
static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
void *tsdbCommitData(STsdbRepo *pRepo) {
if (tsdbStartCommit(pRepo) < 0) {
tsdbError("vgId:%d failed to commit data while startting to commit since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
tsdbStartCommit(pRepo);
// Commit to update meta file
if (tsdbCommitMeta(pRepo) < 0) {
......@@ -138,11 +135,15 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateMFile(&mf, true) < 0) {
tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
} else {
tsdbInitMFileEx(&mf, pOMFile);
if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
}
......@@ -154,12 +155,20 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
if (pAct->act == TSDB_UPDATE_META) {
pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tsdbCloseMFile(&mf);
tsdbApplyMFileChange(&mf, pOMFile);
// TODO: need to reload metaCache
return -1;
}
} else if (pAct->act == TSDB_DROP_META) {
if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tsdbCloseMFile(&mf);
tsdbApplyMFileChange(&mf, pOMFile);
// TODO: need to reload metaCache
return -1;
}
} else {
......@@ -168,6 +177,9 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) {
}
if (tsdbUpdateMFileHeader(&mf) < 0) {
tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
tsdbApplyMFileChange(&mf, pOMFile);
// TODO: need to reload metaCache
return -1;
}
......@@ -208,6 +220,8 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) {
pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey,
pRtn->minFid, pRtn->midFid, pRtn->maxFid);
}
static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) {
......@@ -238,7 +252,7 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void
tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid));
if (pRecord != NULL) {
pMFile->info.tombSize += pRecord->size;
pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
} else {
pMFile->info.nRecords++;
}
......@@ -253,7 +267,7 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
if (pRecord == NULL) {
tsdbError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid);
return -1;
}
......@@ -264,11 +278,11 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
void *pBuf = buf;
tsdbEncodeKVRecord(&pBuf, &rInfo);
if (tsdbAppendMFile(pMFile, buf, POINTER_DISTANCE(pBuf, buf), NULL) < 0) {
if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) {
return -1;
}
pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf));
pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord));
pMFile->info.nDels++;
pMFile->info.nRecords--;
pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
......@@ -302,7 +316,12 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) {
// Skip expired memory data and expired FSET
tsdbSeekCommitIter(&commith, commith.rtn.minKey);
while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
if (pSet->fid >= commith.rtn.minFid) break;
if (pSet->fid < commith.rtn.minFid) {
tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
} else {
break;
}
}
// Loop to commit to each file
......@@ -349,7 +368,7 @@ static int tsdbCommitTSData(STsdbRepo *pRepo) {
return 0;
}
static int tsdbStartCommit(STsdbRepo *pRepo) {
static void tsdbStartCommit(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0);
......@@ -360,7 +379,6 @@ static int tsdbStartCommit(STsdbRepo *pRepo) {
tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd);
pRepo->code = TSDB_CODE_SUCCESS;
return 0;
}
static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
......@@ -413,14 +431,18 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
if (pIter->pTable == NULL) continue;
if (tsdbCommitToTable(pCommith, tid) < 0) {
// TODO: revert the file change
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
}
}
if (tsdbWriteBlockIdx(pCommith) < 0) {
tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
tsdbCloseCommitFile(pCommith, true);
// revert the file change
tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
return -1;
}
......@@ -674,7 +696,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
TSDB_RUNLOCK_TABLE(pIter->pTable);
if (tsdbWriteBlockInfo(pCommith) < 0) return -1;
if (tsdbWriteBlockInfo(pCommith) < 0) {
tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
return -1;
}
return 0;
}
......@@ -926,6 +952,8 @@ static int tsdbWriteBlockIdx(SCommitH *pCommih) {
if (nidx <= 0) {
// All data are deleted
pHeadf->info.offset = 0;
pHeadf->info.len = 0;
return 0;
}
......@@ -1227,7 +1255,6 @@ static void tsdbResetCommitFile(SCommitH *pCommith) {
}
static void tsdbResetCommitTable(SCommitH *pCommith) {
tdResetDataCols(pCommith->pDataCols);
taosArrayClear(pCommith->aSubBlk);
taosArrayClear(pCommith->aSupBlk);
pCommith->pTable = NULL;
......@@ -1256,6 +1283,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
}
tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet),
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
} else {
pCommith->isRFileSet = false;
}
......@@ -1266,6 +1296,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)));
if (tsdbCreateDFileSet(pWSet, true) < 0) {
tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo),
TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno));
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
}
......@@ -1274,6 +1306,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
pCommith->isDFileSame = false;
pCommith->isLFileSame = false;
tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet),
TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet));
} else {
did.level = TSDB_FSET_LEVEL(pSet);
did.id = TSDB_FSET_ID(pSet);
......@@ -1285,6 +1320,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith);
tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD);
if (tsdbCreateDFile(pWHeadf, true) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf),
tstrerror(terrno));
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
return -1;
......@@ -1296,7 +1334,10 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith);
tsdbInitDFileEx(pWDataf, pRDataf);
if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) {
tsdbCloseDFile(pWHeadf);
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
tsdbCloseAndUnsetFSet(&(pCommith->readh));
......@@ -1313,6 +1354,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
pCommith->isLFileSame = true;
if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
......@@ -1325,6 +1369,9 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
pCommith->isLFileSame = false;
if (tsdbCreateDFile(pWLastf, true) < 0) {
tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
tstrerror(terrno));
tsdbCloseDFileSet(pWSet);
tsdbRemoveDFile(pWHeadf);
if (pCommith->isRFileSet) {
......@@ -1360,7 +1407,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
if (pBlock->last) {
if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true;
} else {
if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true;
}
}
......@@ -1373,12 +1420,16 @@ static int tsdbApplyRtn(STsdbRepo *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo);
SDFileSet *pSet;
// Get retentioni snapshot
// Get retention snapshot
tsdbGetRtnSnap(pRepo, &rtn);
tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
while ((pSet = tsdbFSIterNext(&fsiter))) {
if (pSet->fid < rtn.minFid) continue;
if (pSet->fid < rtn.minFid) {
tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
continue;
}
if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) {
return -1;
......@@ -1392,10 +1443,13 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
SDiskID did;
SDFileSet nSet;
STsdbFS * pfs = REPO_FS(pRepo);
int level;
ASSERT(pSet->fid >= pRtn->minFid);
tfsAllocDisk(tsdbGetFidLevel(pSet->fid, pRtn), &(did.level), &(did.id));
level = tsdbGetFidLevel(pSet->fid, pRtn);
tfsAllocDisk(level, &(did.level), &(did.id));
if (did.level == TFS_UNDECIDED_LEVEL) {
terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
return -1;
......@@ -1406,12 +1460,17 @@ static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs));
if (tsdbCopyDFileSet(pSet, &nSet) < 0) {
tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid,
TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno));
return -1;
}
if (tsdbUpdateDFileSet(pfs, &nSet) < 0) {
return -1;
}
tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid,
TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id);
} else {
// On a correct level
if (tsdbUpdateDFileSet(pfs, pSet) < 0) {
......
src/tsdb/src/tsdbFS.c
浏览文件 @ e7a52dfb
......@@ -701,6 +701,8 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
int64_t maxBufSize = 0;
SMFInfo minfo;
taosHashEmpty(pfs->metaCache);
// No meta file, just return
if (pfs->cstatus->pmf == NULL) return 0;
......@@ -718,6 +720,12 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
while (true) {
int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
if (tsize == 0) break;
if (tsize < 0) {
tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
if (tsize < sizeof(SKVRecord)) {
tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
TSDB_FILE_FULL_NAME(pMFile));
......@@ -840,7 +848,7 @@ static int tsdbScanRootDir(STsdbRepo *pRepo) {
continue;
}
if (tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
continue;
}
......
src/tsdb/src/tsdbFile.c
浏览文件 @ e7a52dfb
......@@ -33,7 +33,7 @@ static int tsdbRollBackDFile(SDFile *pDFile);
void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
char fname[TSDB_FILENAME_LEN];
TSDB_FILE_SET_CLOSED(pMFile);
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
memset(&(pMFile->info), 0, sizeof(pMFile->info));
pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
......@@ -42,7 +42,7 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
}
void tsdbInitMFileEx(SMFile *pMFile, SMFile *pOMFile) {
void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
*pMFile = *pOMFile;
TSDB_FILE_SET_CLOSED(pMFile);
}
......@@ -201,6 +201,7 @@ int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) {
tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pMFile));
pRepo->state |= TSDB_STATE_BAD_META;
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
return 0;
}
......@@ -232,6 +233,7 @@ int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) {
tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
pRepo->state |= TSDB_STATE_BAD_META;
TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return 0;
} else {
......@@ -293,6 +295,8 @@ static int tsdbRollBackMFile(SMFile *pMFile) {
void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) {
char fname[TSDB_FILENAME_LEN];
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK);
TSDB_FILE_SET_CLOSED(pDFile);
memset(&(pDFile->info), 0, sizeof(pDFile->info));
......@@ -439,6 +443,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile));
pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
return 0;
}
......@@ -470,6 +475,7 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return 0;
} else {
......
src/tsdb/src/tsdbMemTable.c
浏览文件 @ e7a52dfb
......@@ -216,11 +216,13 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
}
int tsdbAsyncCommit(STsdbRepo *pRepo) {
if (pRepo->mem == NULL) return 0;
tsem_wait(&(pRepo->readyToCommit));
ASSERT(pRepo->imem == NULL);
if (pRepo->mem == NULL) {
tsem_post(&(pRepo->readyToCommit));
return 0;
}
if (pRepo->code != TSDB_CODE_SUCCESS) {
tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno));
......
src/tsdb/src/tsdbRead.c
浏览文件 @ e7a52dfb
......@@ -1176,13 +1176,18 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
}
static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
int32_t numOfCols, STable* pTable) {
int32_t numOfCols, STable* pTable, STSchema* pSchema) {
char* pData = NULL;
// the schema version info is embeded in SDataRow
STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
int32_t numOfRowCols = schemaNCols(pSchema);
int32_t numOfRowCols = 0;
if (pSchema == NULL) {
pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
numOfRowCols = schemaNCols(pSchema);
} else {
numOfRowCols = schemaNCols(pSchema);
}
int32_t i = 0, j = 0;
while(i < numOfCols && j < numOfRowCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
......@@ -1199,10 +1204,38 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
if (pSchema->columns[j].colId == pColInfo->info.colId) {
void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pData, value, varDataTLen(value));
} else {
memcpy(pData, value, pColInfo->info.bytes);
switch (pColInfo->info.type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
memcpy(pData, value, varDataTLen(value));
break;
case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT:
*(uint8_t *)pData = *(uint8_t *)value;
break;
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT:
*(uint16_t *)pData = *(uint16_t *)value;
break;
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT:
*(uint32_t *)pData = *(uint32_t *)value;
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
*(uint64_t *)pData = *(uint64_t *)value;
break;
case TSDB_DATA_TYPE_FLOAT:
SET_FLOAT_PTR(pData, value);
break;
case TSDB_DATA_TYPE_DOUBLE:
SET_DOUBLE_PTR(pData, value);
break;
default:
memcpy(pData, value, pColInfo->info.bytes);
}
j++;
......@@ -1401,6 +1434,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// compared with the data from in-memory buffer, to generate the correct timestamp array list
int32_t numOfRows = 0;
int16_t rv = -1;
STSchema* pSchema = NULL;
int32_t pos = cur->pos;
cur->win = TSWINDOW_INITIALIZER;
......@@ -1429,7 +1465,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
if (rv != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
rv = dataRowVersion(row);
}
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
......@@ -1442,7 +1483,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
if (pCfg->update) {
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
if (rv != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
rv = dataRowVersion(row);
}
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
......@@ -1987,6 +2033,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
int16_t rv = -1;
STSchema* pSchema = NULL;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
......@@ -2007,7 +2055,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
}
win->ekey = key;
copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable);
if (rv != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
rv = dataRowVersion(row);
}
copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema);
if (++numOfRows >= maxRowsToRead) {
moveToNextRowInMem(pCheckInfo);
......@@ -2090,7 +2142,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
return false;
}
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj);
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL);
tfree(pRow);
// update the last key value
......@@ -2164,7 +2216,7 @@ bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) {
return false;
}
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj);
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL);
tfree(pRow);
// update the last key value
......
src/tsdb/src/tsdbSync.c
浏览文件 @ e7a52dfb
......@@ -85,6 +85,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
pRepo->state = TSDB_STATE_OK;
tsdbInitSyncH(&synch, pRepo, socketFd);
tsem_wait(&(pRepo->readyToCommit));
tsdbStartFSTxn(pRepo, 0, 0);
if (tsdbSyncRecvMeta(&synch) < 0) {
......@@ -98,6 +99,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
}
tsdbEndFSTxn(pRepo);
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
// Reload file change
......@@ -107,6 +109,7 @@ int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
_err:
tsdbEndFSTxnWithError(REPO_FS(pRepo));
tsem_post(&(pRepo->readyToCommit));
tsdbDestroySyncH(&synch);
return -1;
}
......@@ -191,7 +194,8 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) {
return 0;
}
if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0) {
if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0 ||
TSDB_FILE_IS_BAD(pLMFile)) {
// Local has no meta file or has a different meta file, need to copy from remote
pSynch->mfChanged = true;
......@@ -409,7 +413,8 @@ static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) {
pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
pLSet = tsdbFSIterNext(&fsiter);
} else {
if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf)) {
if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) &&
tsdbFSetIsOk(pLSet)) {
// Just keep local files and notify remote not to send
tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid);
......
src/util/src/tconfig.c
浏览文件 @ e7a52dfb
......@@ -134,6 +134,11 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
wordfree(&full_path);
char tmp[1025] = {0};
if (realpath(option, tmp) != NULL) {
strcpy(option, tmp);
}
int code = taosMkDir(option, 0755);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
......
src/util/src/tref.c
浏览文件 @ e7a52dfb
......@@ -24,22 +24,22 @@
#define TSDB_REF_STATE_DELETED 2
typedef struct SRefNode {
struct SRefNode *prev; // previous node
struct SRefNode *prev; // previous node
struct SRefNode *next; // next node
void *p; // pointer to resource protected,
void *p; // pointer to resource protected,
int64_t rid; // reference ID
int32_t count; // number of references
int removed; // 1: removed
int removed; // 1: removed
} SRefNode;
typedef struct {
SRefNode **nodeList; // array of SRefNode linked list
int state; // 0: empty, 1: active; 2: deleted
int rsetId; // refSet ID, global unique
int rsetId; // refSet ID, global unique
int64_t rid; // increase by one for each new reference
int max; // mod
int max; // mod
int32_t count; // total number of SRefNodes in this set
int64_t *lockedBy;
int64_t *lockedBy;
void (*fp)(void *);
} SRefSet;
......@@ -62,9 +62,9 @@ int taosOpenRef(int max, void (*fp)(void *))
SRefSet *pSet;
int64_t *lockedBy;
int i, rsetId;
pthread_once(&tsRefModuleInit, taosInitRefModule);
nodeList = calloc(sizeof(SRefNode *), (size_t)max);
if (nodeList == NULL) {
terrno = TSDB_CODE_REF_NO_MEMORY;
......@@ -79,12 +79,12 @@ int taosOpenRef(int max, void (*fp)(void *))
}
pthread_mutex_lock(&tsRefMutex);
for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId
if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
}
}
if (i < TSDB_REF_OBJECTS) {
rsetId = tsNextId;
......@@ -105,7 +105,7 @@ int taosOpenRef(int max, void (*fp)(void *))
free (nodeList);
free (lockedBy);
uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
}
}
pthread_mutex_unlock(&tsRefMutex);
......@@ -127,7 +127,7 @@ int taosCloseRef(int rsetId)
pthread_mutex_lock(&tsRefMutex);
if (pSet->state == TSDB_REF_STATE_ACTIVE) {
if (pSet->state == TSDB_REF_STATE_ACTIVE) {
pSet->state = TSDB_REF_STATE_DELETED;
deleted = 1;
uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
......@@ -142,7 +142,7 @@ int taosCloseRef(int rsetId)
return 0;
}
int64_t taosAddRef(int rsetId, void *p)
int64_t taosAddRef(int rsetId, void *p)
{
int hash;
SRefNode *pNode;
......@@ -163,7 +163,7 @@ int64_t taosAddRef(int rsetId, void *p)
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
}
pNode = calloc(sizeof(SRefNode), 1);
if (pNode == NULL) {
terrno = TSDB_CODE_REF_NO_MEMORY;
......@@ -173,7 +173,7 @@ int64_t taosAddRef(int rsetId, void *p)
rid = atomic_add_fetch_64(&pSet->rid, 1);
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode->p = p;
pNode->rid = rid;
pNode->count = 1;
......@@ -187,16 +187,16 @@ int64_t taosAddRef(int rsetId, void *p)
taosUnlockList(pSet->lockedBy+hash);
return rid;
return rid;
}
int taosRemoveRef(int rsetId, int64_t rid)
int taosRemoveRef(int rsetId, int64_t rid)
{
return taosDecRefCount(rsetId, rid, 1);
}
// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
void *taosAcquireRef(int rsetId, int64_t rid)
void *taosAcquireRef(int rsetId, int64_t rid)
{
int hash;
SRefNode *pNode;
......@@ -204,7 +204,7 @@ void *taosAcquireRef(int rsetId, int64_t rid)
void *p = NULL;
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
//uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return NULL;
}
......@@ -223,7 +223,7 @@ void *taosAcquireRef(int rsetId, int64_t rid)
terrno = TSDB_CODE_REF_ID_REMOVED;
return NULL;
}
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
......@@ -233,8 +233,8 @@ void *taosAcquireRef(int rsetId, int64_t rid)
if (pNode->rid == rid) {
break;
}
pNode = pNode->next;
pNode = pNode->next;
}
if (pNode) {
......@@ -258,7 +258,7 @@ void *taosAcquireRef(int rsetId, int64_t rid)
return p;
}
int taosReleaseRef(int rsetId, int64_t rid)
int taosReleaseRef(int rsetId, int64_t rid)
{
return taosDecRefCount(rsetId, rid, 0);
}
......@@ -280,6 +280,7 @@ void *taosIterateRef(int rsetId, int64_t rid) {
return NULL;
}
void *newP = NULL;
pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
......@@ -289,52 +290,68 @@ void *taosIterateRef(int rsetId, int64_t rid) {
return NULL;
}
int hash = 0;
if (rid > 0) {
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
do {
newP = NULL;
int hash = 0;
if (rid > 0) {
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->rid == rid) break;
pNode = pNode->next;
}
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->rid == rid) break;
pNode = pNode->next;
}
if (pNode == NULL) {
uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
taosUnlockList(pSet->lockedBy+hash);
taosDecRsetCount(pSet);
return NULL;
}
if (pNode == NULL) {
uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
taosUnlockList(pSet->lockedBy+hash);
return NULL;
// rid is there
pNode = pNode->next;
// check first place
while (pNode) {
if (!pNode->removed) break;
pNode = pNode->next;
}
if (pNode == NULL) {
taosUnlockList(pSet->lockedBy+hash);
hash++;
}
}
// rid is there
pNode = pNode->next;
if (pNode == NULL) {
taosUnlockList(pSet->lockedBy+hash);
hash++;
if (pNode == NULL) {
for (; hash < pSet->max; ++hash) {
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
if (pNode) {
// check first place
while (pNode) {
if (!pNode->removed) break;
pNode = pNode->next;
}
if (pNode) break;
}
taosUnlockList(pSet->lockedBy+hash);
}
}
}
if (pNode == NULL) {
for (; hash < pSet->max; ++hash) {
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
if (pNode) break;
if (pNode) {
pNode->count++; // acquire it
newP = pNode->p;
taosUnlockList(pSet->lockedBy+hash);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
} else {
uTrace("rsetId:%d the list is over", rsetId);
}
}
void *newP = NULL;
if (pNode) {
pNode->count++; // acquire it
newP = pNode->p;
taosUnlockList(pSet->lockedBy+hash);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
} else {
uTrace("rsetId:%d the list is over", rsetId);
}
if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
if (pNode) rid = pNode->rid;
} while (newP && pNode->removed);
taosDecRsetCount(pSet);
......@@ -350,22 +367,22 @@ int taosListRef() {
for (int i = 0; i < TSDB_REF_OBJECTS; ++i) {
pSet = tsRefSetList + i;
if (pSet->state == TSDB_REF_STATE_EMPTY)
if (pSet->state == TSDB_REF_STATE_EMPTY)
continue;
uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
for (int j=0; j < pSet->max; ++j) {
pNode = pSet->nodeList[j];
while (pNode) {
uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
pNode = pNode->next;
num++;
}
}
}
}
}
pthread_mutex_unlock(&tsRefMutex);
......@@ -397,16 +414,16 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
}
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->rid == rid)
break;
pNode = pNode->next;
pNode = pNode->next;
}
if (pNode) {
......@@ -417,18 +434,18 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
if (pNode->prev) {
pNode->prev->next = pNode->next;
} else {
pSet->nodeList[hash] = pNode->next;
pSet->nodeList[hash] = pNode->next;
}
if (pNode->next) {
pNode->next->prev = pNode->prev;
}
pNode->next->prev = pNode->prev;
}
released = 1;
} else {
uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid);
}
} else {
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
code = -1;
}
......@@ -437,11 +454,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
if (released) {
uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
(*pSet->fp)(pNode->p);
(*pSet->fp)(pNode->p);
free(pNode);
taosDecRsetCount(pSet);
}
}
return code;
}
......@@ -493,5 +510,5 @@ static void taosDecRsetCount(SRefSet *pSet) {
}
pthread_mutex_unlock(&tsRefMutex);
}
}
src/util/src/tsocket.c
浏览文件 @ e7a52dfb
......@@ -32,14 +32,27 @@ int32_t taosGetFqdn(char *fqdn) {
struct addrinfo hints = {0};
struct addrinfo *result = NULL;
#ifdef __APPLE__
// on macosx, hostname -f has the form of xxx.local
// which will block getaddrinfo for a few seconds if AI_CANONNAME is set
// thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return
// immediately
hints.ai_family = AF_INET;
#else // __APPLE__
hints.ai_flags = AI_CANONNAME;
#endif // __APPLE__
int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
return -1;
}
#ifdef __APPLE__
// refer to comments above
strcpy(fqdn, hostname);
#else // __APPLE__
strcpy(fqdn, result->ai_canonname);
#endif // __APPLE__
freeaddrinfo(result);
return 0;
}
......@@ -286,7 +299,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
int32_t bufSize = 1024 * 1024;
sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sockFd <= 2) {
uError("failed to open the socket: %d (%s)", errno, strerror(errno));
taosCloseSocketNoCheck(sockFd);
......
src/util/src/ttimer.c
浏览文件 @ e7a52dfb
......@@ -188,11 +188,7 @@ static void removeTimer(uintptr_t id) {
}
static int64_t getMonotonicMs(void) {
#ifdef WINDOWS
return (int64_t) getMonotonicUs() / 1000;
#else
return taosGetTimestampMs();
#endif
}
static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
......
src/vnode/src/vnodeCfg.c
浏览文件 @ e7a52dfb
......@@ -34,6 +34,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock;
pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision;
pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
pVnode->tsdbCfg.update = vnodeMsg->cfg.update;
pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow;
pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
......@@ -227,6 +228,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
vnodeMsg.cfg.quorum = (int8_t)quorum->valueint;
cJSON *update = cJSON_GetObjectItem(root, "update");
if (!update || update->type != cJSON_Number) {
vError("vgId: %d, failed to read %s, update not found", pVnode->vgId, file);
vnodeMsg.cfg.update = 0;
vnodeMsg.cfg.vgCfgVersion = 0;
} else {
vnodeMsg.cfg.update = (int8_t)update->valueint;
}
cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow");
if (!cacheLastRow || cacheLastRow->type != cJSON_Number) {
vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file);
......@@ -325,6 +335,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica);
len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals);
len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum);
len += snprintf(content + len, maxLen - len, " \"update\": %d,\n", pMsg->cfg.update);
len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow);
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {
......
src/vnode/src/vnodeMain.c
浏览文件 @ e7a52dfb
......@@ -194,12 +194,14 @@ int32_t vnodeOpen(int32_t vgId) {
int32_t code = vnodeReadCfg(pVnode);
if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId);
vnodeCleanUp(pVnode);
return code;
return 0;
}
code = vnodeReadVersion(pVnode);
if (code != TSDB_CODE_SUCCESS) {
pVnode->version = 0;
vError("vgId:%d, failed to read file version, generate it from data file", pVnode->vgId);
// Allow vnode start even when read file version fails, set file version as wal version or zero
// vnodeCleanUp(pVnode);
......
tests/examples/c/epoll.c
浏览文件 @ e7a52dfb
......@@ -38,6 +38,8 @@
#include <string.h>
#include <arpa/inet.h>
#include <libgen.h>
#include <locale.h>
#include <netdb.h>
#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__)
#define A(statement, fmt, ...) do { \
......@@ -56,6 +58,8 @@
##__VA_ARGS__); \
} while (0)
#include "os.h"
typedef struct ep_s ep_t;
struct ep_s {
int ep;
......@@ -214,7 +218,6 @@ static void* routine(void* arg) {
A(0==pthread_mutex_lock(&ep->lock), "");
ep->wakenup = 0;
A(0==pthread_mutex_unlock(&ep->lock), "");
D("........");
continue;
}
A(ev->data.ptr, "internal logic error");
......
tests/pytest/functions/function_stddev_td2555.py 0 → 100644
浏览文件 @ e7a52dfb
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
import numpy as np
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.rowNum = 100
self.ts = 1537146000000
self.clist1 = []
self.clist2 = []
self.clist3 = []
self.clist4 = []
self.clist5 = []
self.clist6 = []
def getData(self):
for i in range(tdSql.queryRows):
for j in range(6):
exec('self.clist{}.append(tdSql.queryResult[i][j+1])'.format(j+1))
def run(self):
tdSql.prepare()
tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
col7 bool, col8 binary(20), col9 nchar(20)) tags(cid int,gbid binary(20),loc nchar(20))''')
tdSql.execute("create table test1 using test tags(1,'beijing','北京')")
tdSql.execute("create table test2 using test tags(2,'shanghai','深圳')")
tdSql.execute("create table test3 using test tags(2,'shenzhen','深圳')")
tdSql.execute("create table test4 using test tags(1,'shanghai','上海')")
for j in range(4):
for i in range(self.rowNum):
tdSql.execute("insert into test%d values(now-%dh, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')"
% (j+1,i, i + 1, i + 1, i + 1, i + 1, i + i * 0.1, i * 1.5, i % 2, i + 1, i + 1))
# stddev verifacation
tdSql.error("select stddev(ts) from test")
tdSql.error("select stddev(col7) from test")
tdSql.error("select stddev(col8) from test")
tdSql.error("select stddev(col9) from test")
con_list = [
' where cid = 1 and ts >=now - 1d and ts <now',
" where gbid = 'beijing' and ts >=now - 1d and ts <now",
' '
]
for condition in con_list:
tdSql.query("select * from test %s"%(condition))
self.getData()
for i in range(6):
exec('tdSql.query("select stddev(col{}) from test {}")'.format(i+1,condition))
exec('tdSql.checkData(0, 0, np.std(self.clist{}))'.format(i+1))
exec('self.clist{}.clear()'.format(i+1))
print('step 2')
con_group_list = {
' cid = 2 and ts >=now - 1d and ts <now group by tbname':2,
" loc = '深圳' and ts >=now - 1d and ts <now group by tbname " :2 ,
" gbid = 'shanghai' and ts >=now - 1d and ts <now group by cid " :2
}
result = [6.922186552,6.922186552,6.922186552,6.922186552,7.614405212,10.383279829]
for key,value in con_group_list.items():
for i in range(6):
exec('tdSql.query("select stddev(col{}) from test where {}")'.format(i+1,key))
for j in range(value):
tdSql.checkData(j, 0, result[i])
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
tests/pytest/insert/retentionpolicy.py
浏览文件 @ e7a52dfb
......@@ -44,7 +44,8 @@ class TDTestRetetion:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows)
os.system("sudo timedatectl set-ntp true")
time.sleep(40)
os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1)))
time.sleep(5)
tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args)
def run(self):
......@@ -63,7 +64,7 @@ class TDTestRetetion:
tdLog.info("=============== step2")
tdDnodes.stop(1)
os.system("sudo timedatectl set-ntp false")
os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48)))
tdDnodes.start(1)
cmd = 'insert into test values(now,5);'
tdDnodes.stop(1)
......@@ -79,7 +80,7 @@ class TDTestRetetion:
self.checkRows(5,cmd)
tdLog.info("=============== step3")
tdDnodes.stop(1)
os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48)))
tdDnodes.start(1)
tdLog.info(cmd)
tdSql.execute(cmd)
......@@ -99,18 +100,19 @@ class TDTestRetetion:
tdLog.info(cmd)
tdSql.execute(cmd)
self.queryRows=tdSql.query('select * from test')
self.checkRows(7,cmd)
self.checkRows(5,cmd)
tdLog.info("=============== step5")
tdDnodes.stop(1)
tdDnodes.start(1)
cmd='select * from test where ts > now-1d'
self.queryRows=tdSql.query('select * from test where ts > now-1d')
self.checkRows(1,cmd)
self.checkRows(2,cmd)
def stop(self):
os.system("sudo timedatectl set-ntp true")
time.sleep(40)
os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1)))
time.sleep(5)
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
tests/pytest/pytest_2.sh
浏览文件 @ e7a52dfb
# update
#python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
python3 ./test.py -f update/append_commit_data.py
python3 ./test.py -f update/append_commit_last-0.py
python3 ./test.py -f update/append_commit_last.py
#python3 ./test.py -f update/merge_commit_data.py
#python3 ./test.py -f update/merge_commit_data-0.py
#python3 ./test.py -f update/merge_commit_data2.py
#python3 ./test.py -f update/merge_commit_data2_update0.py
#python3 ./test.py -f update/merge_commit_last-0.py
#python3 ./test.py -f update/merge_commit_last.py
#python3 ./test.py -f update/bug_td2279.py
python3 ./test.py -f update/merge_commit_data.py
python3 ./test.py -f update/merge_commit_data-0.py
python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_data2_update0.py
python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py
python3 ./test.py -f update/bug_td2279.py
# wal
python3 ./test.py -f wal/addOldWalTest.py
......
tests/pytest/pytest_3.sh
浏览文件 @ e7a52dfb
......@@ -99,7 +99,7 @@ python3 test.py -f query/queryFillTest.py
python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/lowaTest.py
python3 test.py -f tools/taosdemoTest2.py
#python3 test.py -f tools/taosdemoTest2.py
# subscribe
python3 test.py -f subscribe/singlemeter.py
......
tests/pytest/query/query.py
浏览文件 @ e7a52dfb
......@@ -111,6 +111,18 @@ class TDTestCase:
tdSql.query("select * from tb where c5 = 'true' ")
tdSql.checkRows(5)
# For jira: https://jira.taosdata.com:18080/browse/TD-2850
tdSql.execute("create database 'Test' ")
tdSql.execute("use 'Test' ")
tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)")
tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)")
tdSql.query("select * from tb")
tdSql.checkRows(1)
tdSql.query("select * from tb0")
tdSql.checkRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
tests/pytest/stable/insert.py
浏览文件 @ e7a52dfb
......@@ -47,6 +47,40 @@ class TDTestCase:
tdSql.query("select * from db.st where dev='dev_02'")
tdSql.checkRows(1)
#For: https://jira.taosdata.com:18080/browse/TD-2671
print("==============step3")
tdSql.execute(
"create stable if not exists stb (ts timestamp, tagtype int) tags(dev nchar(50))")
tdSql.execute(
'CREATE TABLE if not exists dev_01 using stb tags("dev_01")')
tdSql.execute(
'CREATE TABLE if not exists dev_02 using stb tags("dev_02")')
print("==============step4")
tdSql.execute(
"""INSERT INTO dev_01(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1),
('2020-05-13 10:00:00.001', 1)
dev_02 VALUES('2020-05-13 10:00:00.001', 1)""")
tdSql.query("select * from db.stb where dev='dev_01'")
tdSql.checkRows(2)
tdSql.query("select * from db.stb where dev='dev_02'")
tdSql.checkRows(1)
tdSql.query("describe db.stb")
tdSql.checkRows(3)
tdSql.execute("alter stable db.stb add tag t1 int")
tdSql.query("describe db.stb")
tdSql.checkRows(4)
tdSql.execute("drop stable db.stb")
tdSql.query("show stables")
tdSql.checkRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
......
tests/pytest/tools/taosdemoTest2.py
浏览文件 @ e7a52dfb
......@@ -31,10 +31,18 @@ class TDTestCase:
def insertDataAndAlterTable(self, threadID):
if(threadID == 0):
os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords))
os.system("yes | taosdemo -t %d -n %d -x" % (self.numberOfTables, self.numberOfRecords))
if(threadID == 1):
print("use test")
tdSql.execute("use test")
while True:
print("query started")
tdSql.query("select * from test.t9")
rows = tdSql.queryRows
print("rows %d" % rows)
if(rows > 0):
break
time.sleep(1)
print("alter table test.meters add column f4 int")
tdSql.execute("alter table test.meters add column f4 int")
print("insert into test.t0 values (now, 1, 2, 3, 4)")
......@@ -46,8 +54,8 @@ class TDTestCase:
t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, ))
t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, ))
t1.start()
time.sleep(2)
t1.start()
time.sleep(2)
t2.start()
t1.join()
t2.join()
......
tests/script/general/parser/dbtbnameValidate.sim
浏览文件 @ e7a52dfb
......@@ -12,24 +12,24 @@ sql create database 'abc123'
sql create database '_ab1234'
sql create database 'ABC123'
sql create database '_ABC123'
sql create database 'aABb123 '
sql create database ' xyz '
sql create database ' XYZ '
sql_error create database 'aABb123 '
sql_error create database ' xyz '
sql_error create database ' XYZ '
sql use 'abc123'
sql use '_ab1234'
sql use 'ABC123'
sql use '_ABC123'
sql use 'aABb123'
sql use ' xyz '
sql use ' XYZ '
sql_error use 'aABb123'
sql_error use ' xyz '
sql_error use ' XYZ '
sql drop database 'abc123'
sql drop database '_ab1234'
sql_error drop database 'ABC123'
sql drop database '_ABC123'
sql drop database 'aABb123'
sql drop database ' xyz '
sql_error drop database 'aABb123'
sql_error drop database ' xyz '
sql_error drop database ' XYZ '
......
tests/script/wtest.bat
浏览文件 @ e7a52dfb
......@@ -44,10 +44,10 @@ echo serverPort 7100 >> %TAOS_CFG%
echo logDir %LOG_DIR% >> %TAOS_CFG%
echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG%
echo numOfLogLines 100000000 >> %TAOS_CFG%
echo rpcDebugFlag 143 >> %TAOS_CFG%
echo rpcDebugFlag 135 >> %TAOS_CFG%
echo tmrDebugFlag 131 >> %TAOS_CFG%
echo cDebugFlag 143 >> %TAOS_CFG%
echo udebugFlag 143 >> %TAOS_CFG%
echo cDebugFlag 135 >> %TAOS_CFG%
echo udebugFlag 135 >> %TAOS_CFG%
echo wal 0 >> %TAOS_CFG%
echo asyncLog 0 >> %TAOS_CFG%
echo locale en_US.UTF-8 >> %TAOS_CFG%
......
tests/test-all.sh
浏览文件 @ e7a52dfb
......@@ -6,15 +6,15 @@ GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
function git_branch {
branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`"
if [ "${branch}" != "" ];then
if [ "${branch}" = "(no branch)" ];then
branch="(`git rev-parse --short HEAD`...)"
fi
branch=(${branch////_})
echo "$branch"
fi
function dohavecore(){
corefile=`find $corepath -mmin 1`
if [ -n "$corefile" ];then
echo 'taosd or taos has generated core'
if [[ $1 == 1 ]];then
exit 8
fi
fi
}
function runSimCaseOneByOne {
while read -r line; do
......@@ -42,6 +42,7 @@ function runSimCaseOneByOne {
# fi
end_time=`date +%s`
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
dohavecore 0
fi
done < $1
}
......@@ -69,14 +70,15 @@ function runSimCaseOneByOnefq {
out_log=`tail -1 out.log `
if [[ $out_log =~ 'failed' ]];then
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"`
cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
else
cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" `
cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
fi
exit 8
fi
end_time=`date +%s`
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
dohavecore 1
fi
done < $1
}
......@@ -105,6 +107,7 @@ function runPyCaseOneByOne {
else
$line > /dev/null 2>&1
fi
dohavecore 0
fi
done < $1
}
......@@ -126,13 +129,14 @@ function runPyCaseOneByOnefq {
end_time=`date +%s`
out_log=`tail -1 pytest-out.log `
if [[ $out_log =~ 'failed' ]];then
cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" `
cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
exit 8
fi
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
else
$line > /dev/null 2>&1
fi
dohavecore 1
fi
done < $1
}
......@@ -140,7 +144,7 @@ totalFailed=0
totalPyFailed=0
tests_dir=`pwd`
corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern`
if [ "$2" != "python" ]; then
echo "### run TSIM test case ###"
cd $tests_dir/script
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或