diff --git a/.gitignore b/.gitignore index 153a71bb7d24ab667f81e79888a6aad1a3bfd61e..1cb4767e7c9447d1fcbc69307da2a632425ec293 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ tests/script/ tests/pytest/ tests/jenkins/ tests/hdfs/ + +*.iml diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 0fdc73080a508fa692746a38808fe017bf085573..8840fd7e823fd21f212739a38ed5cccdc4bb2914 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -1,15 +1,77 @@ 4.0.0 + com.taosdata.jdbc taos-jdbcdriver - 1.0.1 + 1.0.2 + jar + JDBCDriver + https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc TDengine JDBC Driver - - UTF-8 - - + + + + GNU AFFERO GENERAL PUBLIC LICENSE Version 3 + https://github.com/taosdata/TDengine/blob/master/LICENSE + repo + + + + + scm:git:git://github.com/taosdata/TDengine.git + scm:git:git@github.com:taosdata/TDengine.git + https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc + HEAD + + + + + taosdata + support@taosdata.com + https://www.taosdata.com/ + https://www.taosdata.com/ + + + + + UTF-8 + 1.8 + + 3.0.1 + 3.6.0 + 1.6 + 3.1.0 + 2.19.1 + 2.8.2 + 1.6.7 + 2.5.3 + 1.9.5 + + 1.1.2 + 3.5 + + + + + commons-logging + commons-logging + ${commons-logging.version} + + + * + * + + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + @@ -18,7 +80,7 @@ 3.0.0 - + src/main/assembly/assembly-jar.xml @@ -35,18 +97,19 @@ org.apache.maven.plugins maven-compiler-plugin - 2.3.2 + ${maven-compiler-plugin.version} UTF-8 - 1.8 - 1.8 + ${java.version} + ${java.version} true + true org.apache.maven.plugins maven-source-plugin - 2.1.2 + ${maven-source-plugin.version} attach-sources @@ -56,24 +119,101 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + UTF-8 + UTF-8 + UTF-8 + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven-gpg-plugin.version} + + + sign-artifacts + verify + + sign + + + + + + maven-deploy-plugin + ${maven-deploy-plugin.version} + + + default-deploy + deploy + + deploy + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus-staging-maven-plugin.version} + true + + ossrh-td + https://oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-release-plugin + ${maven-release-plugin.version} + + true + false + forked-path + -Dgpg.passphrase=${gpg.passphrase} + + + + org.apache.maven.scm + maven-scm-provider-gitexe + ${maven-scm-provider-gitexe.version} + + + + - - - commons-logging - commons-logging - 1.1.2 - - - * - * - - - - - org.apache.commons - commons-lang3 - 3.5 - - + + + + ossrh-td + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh-td + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..5e4aacb8601a6a0cc3e82d3c2ac7640248c047ef --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java @@ -0,0 +1,423 @@ +package com.taosdata.jdbc; + +import com.taosdata.jdbc.bean.TSDBPreparedParam; + +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * this class is used to precompile the sql of tdengine insert or import ops + */ +public class SavedPreparedStatement { + + private TSDBPreparedStatement tsdbPreparedStatement; + + /** + * sql param List + */ + private List sqlParamList; + + /** + * init param according the sql + */ + private TSDBPreparedParam initPreparedParam; + + /** + * is table name dynamic in the prepared sql + */ + private boolean isTableNameDynamic; + + /** + * insert or import sql template pattern, the template are the following: + * + * insert/import into tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] values(?, ?, ...) (?, ?, ...) + * + * we split it to three part: + * 1. prefix, insert/import + * 2. middle, tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] + * 3. valueList, the content after values, for example (?, ?, ...) (?, ?, ...) + */ + private Pattern sqlPattern = Pattern.compile("(?s)(?i)^\\s*(INSERT|IMPORT)\\s+INTO\\s+((?\\S+)\\s*(\\(.*\\))?\\s+(USING\\s+(?\\S+)\\s+TAGS\\s*\\((?.+)\\))?)\\s*VALUES\\s*(?\\(.*\\)).*"); + + /** + * the raw sql template + */ + private String sql; + + /** + * the prefix part of sql + */ + private String prefix; + + /** + * the middle part of sql + */ + private String middle; + + private int middleParamSize; + + /** + * the valueList part of sql + */ + private String valueList; + + private int valueListSize; + + /** + * default param value + */ + private static final String DEFAULT_VALUE= "NULL"; + + private static final String PLACEHOLDER= "?"; + + private String tableName; + + /** + * is the parameter add to batch list + */ + private boolean isAddBatch; + + public SavedPreparedStatement(String sql, TSDBPreparedStatement tsdbPreparedStatement)throws SQLException { + this.sql = sql; + this.tsdbPreparedStatement = tsdbPreparedStatement; + this.sqlParamList = new ArrayList<>(); + + parsePreparedParam(this.sql); + } + + /** + * parse the init param according the sql param + * @param sql + */ + private void parsePreparedParam(String sql)throws SQLException{ + + Matcher matcher = sqlPattern.matcher(sql); + + if(matcher.find()){ + + tableName = matcher.group("tablename"); + + if (tableName != null && PLACEHOLDER.equals(tableName)){ + // the table name is dynamic + this.isTableNameDynamic = true; + } + + prefix = matcher.group(1); + middle = matcher.group(2); + valueList = matcher.group("valueList"); + + if(middle != null && !"".equals(middle)){ + middleParamSize = parsePlaceholder(middle); + } + + if(valueList != null && !"".equals(valueList)){ + valueListSize = parsePlaceholder(valueList); + } + + initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); + + }else{ + // not match + throw new SQLException(TSDBConstants.WrapErrMsg("the sql is not complete!")); + } + + } + + private TSDBPreparedParam initDefaultParam(String tableName, int middleParamSize, int valueListSize){ + + TSDBPreparedParam tsdbPreparedParam = new TSDBPreparedParam(tableName); + + tsdbPreparedParam.setMiddleParamList(getDefaultParamList(middleParamSize)); + + tsdbPreparedParam.setValueList(getDefaultParamList(valueListSize)); + + return tsdbPreparedParam; + } + + /** + * generate the default param value list + * @param paramSize + * @return + */ + private List getDefaultParamList(int paramSize){ + + List paramList = new ArrayList<>(paramSize); + if (paramSize > 0){ + for (int i = 0; i < paramSize; i++){ + paramList.add(i, DEFAULT_VALUE); + } + } + + return paramList; + } + + /** + * calculate the placeholder num + * @param value + * @return + */ + private int parsePlaceholder(String value){ + + Pattern pattern = Pattern.compile("[?]"); + + Matcher matcher = pattern.matcher(value); + + int result = 0; + while (matcher.find()){ + result++; + } + return result; + } + + /** + * set current row params + * @param parameterIndex the first parameter is 1, the second is 2, ... + * @param x the parameter value + */ + public void setParam(int parameterIndex, Object x) throws SQLException{ + + int paramSize = this.middleParamSize + this.valueListSize; + + String errorMsg = String.format("the parameterIndex %s out of the range [1, %s]", parameterIndex, this.middleParamSize + this.valueListSize); + + if (parameterIndex < 1 || parameterIndex > paramSize){ + throw new SQLException(TSDBConstants.WrapErrMsg(errorMsg)); + } + + this.isAddBatch = false; //set isAddBatch to false + + if (x == null){ + x = DEFAULT_VALUE; // set default null string + } + + parameterIndex = parameterIndex -1; // start from 0 in param list + + if (this.middleParamSize != 0 && parameterIndex >= 0 && parameterIndex < this.middleParamSize){ + + this.initPreparedParam.setMiddleParam(parameterIndex, x); + return; + } + + if (this.valueListSize != 0 && parameterIndex >= this.middleParamSize && parameterIndex < (this.middleParamSize + this.valueListSize)){ + + this.initPreparedParam.setValueParam(parameterIndex - this.middleParamSize, x); + return; + } + + throw new SQLException(TSDBConstants.WrapErrMsg(errorMsg)); + } + + public void addBatch() { + + addCurrentRowParamToList(); + this.initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); + } + + /** + * add current param to batch list + */ + private void addCurrentRowParamToList(){ + + if (initPreparedParam != null && (this.middleParamSize > 0 || this.valueListSize > 0)){ + this.sqlParamList.add(initPreparedParam); // add current param to batch list + } + this.isAddBatch = true; + } + + + /** + * execute the sql with batch sql + * @return + * @throws SQLException + */ + public int[] executeBatch() throws SQLException { + + int result = executeBatchInternal(); + + return new int[]{result}; + } + + + public int executeBatchInternal() throws SQLException{ + + if (!isAddBatch){ + addCurrentRowParamToList(); // add current param to batch list + } + + //1. generate batch sql + String sql = generateExecuteSql(); + + //2. execute batch sql + int result = executeSql(sql); + + //3. clear batch param list + this.sqlParamList.clear(); + + return result; + } + + /** + * generate the batch sql + * @return + */ + private String generateExecuteSql(){ + + StringBuilder stringBuilder = new StringBuilder(); + + stringBuilder.append(prefix); + stringBuilder.append(" into "); + + if (!isTableNameDynamic){ + // tablename will not need to be replaced + stringBuilder.append(middle); + stringBuilder.append(" values"); + + stringBuilder.append(replaceValueListParam(valueList, sqlParamList)); + + } else{ + // need to replace tablename + + if (sqlParamList.size() > 0 ){ + + TSDBPreparedParam firstPreparedParam = sqlParamList.get(0); + + //replace middle part and value part of first row + String firstRow = replaceMiddleAndValuePart(firstPreparedParam); + stringBuilder.append(firstRow); + + //the first param in the middleParamList is the tableName + String lastTableName = firstPreparedParam.getMiddleParamList().get(0).toString(); + + if (sqlParamList.size() > 1){ + + for (int i = 1; i < sqlParamList.size(); i++){ + TSDBPreparedParam currentParam = sqlParamList.get(i); + String currentTableName = currentParam.getMiddleParamList().get(0).toString(); + if (lastTableName.equalsIgnoreCase(currentTableName)){ + // tablename is same with the last row ,so only need to append the part of value + + String values = replaceTemplateParam(valueList, currentParam.getValueList()); + stringBuilder.append(values); + + } else { + // tablename difference with the last row + //need to replace middle part and value part + String row = replaceMiddleAndValuePart(currentParam); + stringBuilder.append(row); + lastTableName = currentTableName; + } + } + } + + }else{ + + stringBuilder.append(middle); + stringBuilder.append(" values"); + stringBuilder.append(valueList); + } + + } + + return stringBuilder.toString(); + } + + /** + * replace the middle and value part + * @param tsdbPreparedParam + * @return + */ + private String replaceMiddleAndValuePart(TSDBPreparedParam tsdbPreparedParam){ + + StringBuilder stringBuilder = new StringBuilder(" "); + + String middlePart = replaceTemplateParam(middle, tsdbPreparedParam.getMiddleParamList()); + + stringBuilder.append(middlePart); + stringBuilder.append(" values "); + + String valuePart = replaceTemplateParam(valueList, tsdbPreparedParam.getValueList()); + stringBuilder.append(valuePart); + stringBuilder.append(" "); + + return stringBuilder.toString(); + } + + /** + * replace the placeholder of the template with TSDBPreparedParam list + * @param template + * @param sqlParamList + * @return + */ + private String replaceValueListParam(String template, List sqlParamList){ + + StringBuilder stringBuilder = new StringBuilder(); + + if (sqlParamList.size() > 0 ){ + + for (TSDBPreparedParam tsdbPreparedParam : sqlParamList){ + + String tmp = replaceTemplateParam(template, tsdbPreparedParam.getValueList()); + + stringBuilder.append(tmp); + } + + } else { + stringBuilder.append(template); + } + + return stringBuilder.toString(); + } + + /** + * replace the placeholder of the template with paramList + * @param template + * @param paramList + * @return + */ + private String replaceTemplateParam(String template, List paramList){ + + if (paramList.size() > 0){ + + String tmp = template; + + for (int i = 0; i < paramList.size(); ++i) { + + String paraStr = getParamString(paramList.get(i)); + + tmp = tmp.replaceFirst("[" + PLACEHOLDER + "]", paraStr); + + } + + return tmp; + + } else { + return template; + } + + } + + /** + * get the string of param object + * @param paramObj + * @return + */ + private String getParamString(Object paramObj){ + + String paraStr = paramObj.toString(); + if (paramObj instanceof Timestamp || (paramObj instanceof String && !DEFAULT_VALUE.equalsIgnoreCase(paraStr))) { + paraStr = "'" + paraStr + "'"; + } + return paraStr; + } + + + private int executeSql(String sql)throws SQLException{ + + return tsdbPreparedStatement.executeUpdate(sql); + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index bd45fbfb7a25fadaf1e42da63ea1065518d3e53a..674908679c128b54b2a8870080a4a921198dd6eb 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -34,13 +34,54 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat protected String sql; protected ArrayList parameters = new ArrayList(); + //start with insert or import and is case-insensitive + private static Pattern savePattern = Pattern.compile("(?i)^\\s*(insert|import)"); + + // is insert or import + private boolean isSaved; + + private SavedPreparedStatement savedPreparedStatement; + TSDBPreparedStatement(TSDBJNIConnector connecter, String sql) { super(connecter); - this.rawSql = sql; - preprocessSql(); + init(sql); } - public ArrayList getParameters() { + private void init(String sql){ + this.rawSql = sql; + preprocessSql(); + + this.isSaved = isSavedSql(this.rawSql); + if (this.isSaved){ + + try { + this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); + } catch (SQLException e){ + e.printStackTrace(); + } + } + } + + /** + * if the precompiled sql is insert or import + * @param sql + * @return + */ + private boolean isSavedSql(String sql){ + Matcher matcher = savePattern.matcher(sql); + return matcher.find(); + } + + @Override + public int[] executeBatch() throws SQLException { + if (isSaved){ + return this.savedPreparedStatement.executeBatch(); + } else { + return super.executeBatch(); + } + } + + public ArrayList getParameters() { return parameters; } @@ -134,12 +175,21 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public ResultSet executeQuery() throws SQLException { - return super.executeQuery(getNativeSql()); + if (isSaved){ + this.savedPreparedStatement.executeBatchInternal(); + return null; + }else { + return super.executeQuery(getNativeSql()); + } } @Override public int executeUpdate() throws SQLException { - return super.executeUpdate(getNativeSql()); + if (isSaved){ + return this.savedPreparedStatement.executeBatchInternal(); + } else { + return super.executeUpdate(getNativeSql()); + } } @Override @@ -239,20 +289,34 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public void setObject(int parameterIndex, Object x) throws SQLException { - parameters.add(x); + if (isSaved){ + this.savedPreparedStatement.setParam(parameterIndex, x); + }else{ + parameters.add(x); + } } @Override public boolean execute() throws SQLException { - return super.execute(getNativeSql()); + if (isSaved){ + int result = this.savedPreparedStatement.executeBatchInternal(); + return result > 0; + } else { + return super.execute(getNativeSql()); + } } @Override public void addBatch() throws SQLException { - if (this.batchedArgs == null) { - batchedArgs = new ArrayList(); - } - super.addBatch(getNativeSql()); + if (isSaved){ + this.savedPreparedStatement.addBatch(); + }else { + + if (this.batchedArgs == null) { + batchedArgs = new ArrayList(); + } + super.addBatch(getNativeSql()); + } } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java new file mode 100644 index 0000000000000000000000000000000000000000..3d58075c717e89b10637ce3ad933d4d7bcb0cf59 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java @@ -0,0 +1,62 @@ +package com.taosdata.jdbc.bean; + +import java.util.List; + +/** + * tdengine batch insert or import param object + */ +public class TSDBPreparedParam { + + /** + * tableName, if sTable Name is not null, and this is sub table name. + */ + private String tableName; + + /** + * sub middle param list + */ + private List middleParamList; + + /** + * value list + */ + private List valueList; + + public TSDBPreparedParam(String tableName) { + this.tableName = tableName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public List getMiddleParamList() { + return middleParamList; + } + + public void setMiddleParamList(List middleParamList) { + this.middleParamList = middleParamList; + } + + public void setMiddleParam(int parameterIndex, Object x){ + this.middleParamList.set(parameterIndex, x); + } + + public List getValueList() { + return valueList; + } + + public void setValueList(List valueList) { + this.valueList = valueList; + } + + + public void setValueParam(int parameterIndex, Object x){ + this.valueList.set(parameterIndex, x); + } + +}