Skip to content

I
Iotdb

apache / Iotdb
大约 1 年 前同步成功

通知 25
Star 3344
Fork 916
I
Iotdb

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 2a472685 编写于 作者: X Xiangdong Huang 提交者： GitHub
浏览文件
操作

Merge pull request #58 from apache/fix_sonar_bug 

Fix sonar bugs
上级 a1e3a303 bbeb32fd
隐藏空白更改
内联 并排
Showing with 460 addition and 654 deletion
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
浏览文件 @ 2a472685
......@@ -83,7 +83,6 @@ public abstract class AbstractClient {
protected static final String TIMESTAMP_STR = "Time";
protected static final int ISO_DATETIME_LEN = 26;
protected static final String IMPORT_CMD = "import";
protected static final String EXPORT_CMD = "export";
private static final String NEED_NOT_TO_PRINT_TIMESTAMP = "AGGREGATION";
private static final String DEFAULT_TIME_FORMAT = "default";
protected static String timeFormat = DEFAULT_TIME_FORMAT;
......@@ -528,11 +527,7 @@ public abstract class AbstractClient {
}
if (specialCmd.startsWith(SHOW_TIMEZONE)) {
try {
println("Current time zone: " + connection.getTimeZone());
} catch (IoTDBSQLException | TException e) {
handleException(e);
}
showTimeZone(connection);
return OperationResult.CONTINUE_OPER;
}
if (specialCmd.startsWith(SHOW_TIMESTAMP_DISPLAY)) {
......@@ -689,7 +684,7 @@ public abstract class AbstractClient {
if (hasResultSet) {
ResultSet resultSet = statement.getResultSet();
output(resultSet, printToConsole, zoneId);
closeResultSet(resultSet);
}
} catch (Exception e) {
println("Msg: " + e.getMessage());
......@@ -723,27 +718,27 @@ public abstract class AbstractClient {
STOP_OPER, CONTINUE_OPER, NO_OPER
}
private static void printf(String format, Object ... args) {
protected static void printf(String format, Object ... args) {
SCREEN_PRINTER.printf(format, args);
}
private static void print(String msg) {
protected static void print(String msg) {
SCREEN_PRINTER.println(msg);
}
private static void println() {
protected static void println() {
SCREEN_PRINTER.println();
}
private static void println(String msg) {
protected static void println(String msg) {
SCREEN_PRINTER.println(msg);
}
private static void println(Object obj) {
protected static void println(Object obj) {
SCREEN_PRINTER.println(obj);
}
private static void handleException(Exception e) {
protected static void handleException(Exception e) {
if (showException) {
e.printStackTrace(SCREEN_PRINTER);
}
......
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/Client.java
浏览文件 @ 2a472685
......@@ -18,15 +18,10 @@
*/
package org.apache.iotdb.cli.client;
import java.io.IOException;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jline.console.ConsoleReader;
import jline.console.completer.ArgumentCompleter;
import jline.console.completer.Completer;
import jline.console.completer.StringsCompleter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
......@@ -36,38 +31,11 @@ import org.apache.commons.cli.ParseException;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
public class Client extends AbstractClient {
private static final String[] FUNCTION_NAME_LIST = new String[]{"count", "max_time", "min_time",
"max_value",
"min_value", "subsequence_matching", // with parameter(s)
"now()" // without parameter
};
private static final String[] KEY_WORD_LIST = new String[]{"SHOW", "SELECT", "DROP", "UPDATE",
"DELETE", "CREATE",
"INSERT", "INDEX", "TIMESERIES", "TIME", "TIMESTAMP", "VALUES", "FROM", "WHERE", "TO", "ON",
"WITH",
"USING", "AND", "OR", "USER", "ROLE", "EXIT", "QUIT", "IMPORT"};
private static final String[] CONF_NAME_LIST = new String[]{ // set <key>=<value>
"time_display_type", "time_zone", // specific format
"fetch_size", "max_display_num", // integer
"storage group to" // special
};
private static final String[] PARAM_NAME_LIST = new String[]{"datatype", "encoding", // list
"window_length", // integer
};
private static final String[] DATA_TYPE_LIST = new String[]{"BOOLEAN", "INT32", "INT64", "INT96",
"FLOAT",
"DOUBLE", "TEXT", "FIXED_LEN_BYTE_ARRAY", "ENUMS", "BIGDECIMAL"};
private static final String[] ENCODING_LIST = new String[]{"PLAIN", "PLAIN_DICTIONARY", "RLE",
"DIFF", "TS_2DIFF",
"BITMAP", "GORILLA"};
private static CommandLine commandLine;
/**
* IoTDB CLI main function.
......@@ -80,50 +48,63 @@ public class Client extends AbstractClient {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
String[] newArgs = null;
commandLine = null;
String[] newArgs;
if (args == null || args.length == 0) {
System.out.println(
println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) "
+ "-h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
println("For more information, please check the following hint.");
hf.printHelp(SCRIPT_HINT, options, true);
return;
}
init();
newArgs = removePasswordArgs(args);
boolean continues = parseCommandLine(options, newArgs, hf);
if (!continues) {
return;
}
serve();
}
private static boolean parseCommandLine(Options options, String[] newArgs, HelpFormatter hf) {
try {
CommandLineParser parser = new DefaultParser();
commandLine = parser.parse(options, newArgs);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(SCRIPT_HINT, options, true);
return;
return false;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
setMaxDisplayNumber(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
} catch (NumberFormatException e) {
System.out.println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
}
setMaxDisplayNumber(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
}
} catch (ParseException e) {
System.out.println(
println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) "
+ "-h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
println("For more information, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
handleException(e);
return false;
} catch (NumberFormatException e) {
println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
handleException(e);
return false;
}
return true;
}
private static void serve() {
try (ConsoleReader reader = new ConsoleReader()) {
reader.setExpandEvents(false);
String s;
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
......@@ -133,189 +114,58 @@ public class Client extends AbstractClient {
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
try{
IoTDBConnection connection = (IoTDBConnection) DriverManager.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
s = reader.readLine(IOTDB_CLI_PREFIX + "> ", null);
if (s != null) {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length && !isQuit; i++) {
String cmd = cmds[i];
if (cmd != null && !"".equals(cmd.trim())) {
OperationResult result = handleInputCmd(cmd, connection);
switch (result) {
case STOP_OPER:
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
}
if(isQuit) {
break;
}
}
connection.close();
System.out.println(String.format("%s> exit normally.", IOTDB_CLI_PREFIX));
} catch (SQLException e) {
System.out.println(String
.format("%s> %s Host is %s, port is %s.", IOTDB_CLI_PREFIX, e.getMessage(), host,
port));
}
receiveCommands(reader);
} catch (ArgsErrorException e) {
System.out.println(IOTDB_CLI_PREFIX + "> input params error because" + e.getMessage());
println(IOTDB_CLI_PREFIX + "> input params error because" + e.getMessage());
handleException(e);
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
handleException(e);
}
}
/**
* @deprecated this method has been deprecated.
*/
@Deprecated
private static Completer[] getCommandCompleter() {
List<String> candidateStrings = new ArrayList<>();
for (String s : FUNCTION_NAME_LIST) {
if (!s.endsWith("()")) {
candidateStrings.add(s + "(");
} else { // for functions with no parameter, such as now().
candidateStrings.add(s + " ");
}
}
for (String s : KEY_WORD_LIST) {
candidateStrings.add(s + " ");
candidateStrings.add(s.toLowerCase() + " ");
}
StringsCompleter strCompleter = new StringsCompleter(candidateStrings);
ArgumentCompleter.ArgumentDelimiter delim = new ArgumentCompleter.AbstractArgumentDelimiter() {
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
char c = buffer.charAt(pos);
return Character.isWhitespace(c) || c == '(' || c == ')' || c == ',';
}
};
final ArgumentCompleter argCompleter = new ArgumentCompleter(delim, strCompleter);
argCompleter.setStrict(false);
StringsCompleter confCompleter = new StringsCompleter(Arrays.asList(CONF_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter setCompleter = new StringsCompleter(Arrays.asList("set", "show")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && ("set".equals(buffer) || "show".equals(buffer))
? super.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter confPropCompleter = new ArgumentCompleter(setCompleter, confCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
StringsCompleter insertConfCompleter = new StringsCompleter(Arrays.asList("into")) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter insertCompleter = new StringsCompleter(Arrays.asList("insert")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && ("insert".equals(buffer)) ? super
.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter insertPropCompleter = new ArgumentCompleter(insertCompleter,
insertConfCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
private static void receiveCommands(ConsoleReader reader) throws TException, IOException {
try (IoTDBConnection connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password)) {
String s;
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
s = reader.readLine(IOTDB_CLI_PREFIX + "> ", null);
boolean continues = processCmd(s, connection);
if (!continues) {
break;
}
return ret;
}
};
} catch (SQLException e) {
println(String
.format("%s> %s Host is %s, port is %s.", IOTDB_CLI_PREFIX, e.getMessage(), host,
port));
handleException(e);
}
}
StringsCompleter withParamCompleter = new StringsCompleter(Arrays.asList(PARAM_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1) {
int equalsIdx = buffer.indexOf('=');
if (equalsIdx != -1) {
String confName = buffer.substring(0, equalsIdx);
String value = buffer.substring(equalsIdx + 1).toUpperCase();
if (confName.startsWith("encoding")) {
for (String str : ENCODING_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
} else if (confName.startsWith("datatype")) {
for (String str : DATA_TYPE_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
}
return cursor;
}
}
return result;
}
};
ArgumentCompleter withParamPropCompleter = new ArgumentCompleter(delim, withParamCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
private static boolean processCmd(String s, IoTDBConnection connection) {
if (s == null) {
return true;
}
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !"".equals(cmd.trim())) {
OperationResult result = handleInputCmd(cmd, connection);
switch (result) {
case STOP_OPER:
return false;
case CONTINUE_OPER:
continue;
default:
break;
}
return ret;
}
};
withParamPropCompleter.setStrict(false);
return new Completer[]{confPropCompleter, insertPropCompleter, withParamPropCompleter,
argCompleter};
}
return true;
}
}
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/WinClient.java
浏览文件 @ 2a472685
......@@ -31,9 +31,12 @@ import org.apache.commons.cli.ParseException;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
public class WinClient extends AbstractClient {
private static CommandLine commandLine;
/**
* main function.
*
......@@ -44,12 +47,12 @@ public class WinClient extends AbstractClient {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
String[] newArgs = null;
commandLine = null;
String[] newArgs;
if (args == null || args.length == 0) {
System.out.println("Require more params input, please check the following hint.");
println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
......@@ -58,33 +61,57 @@ public class WinClient extends AbstractClient {
newArgs = removePasswordArgs(args);
boolean continues = parseCommandLine(options, newArgs, hf);
if (!continues) {
return;
}
serve();
}
private static String readPassword() {
Console c = System.console();
if (c == null) { // IN ECLIPSE IDE
print(IOTDB_CLI_PREFIX + "> please input password: ");
Scanner scanner = new Scanner(System.in);
return scanner.nextLine();
} else { // Outside Eclipse IDE
return new String(c.readPassword(IOTDB_CLI_PREFIX + "> please input password: "));
}
}
private static boolean parseCommandLine(Options options, String[] newArgs, HelpFormatter hf) {
try {
CommandLineParser parser = new DefaultParser();
commandLine = parser.parse(options, newArgs);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
return false;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
maxPrintRowCount = Integer.valueOf(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
if (maxPrintRowCount < 0) {
maxPrintRowCount = Integer.MAX_VALUE;
}
} catch (NumberFormatException e) {
System.out.println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
maxPrintRowCount = Integer.valueOf(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
if (maxPrintRowCount < 0) {
maxPrintRowCount = Integer.MAX_VALUE;
}
}
} catch (ParseException e) {
System.out.println("Require more params input, please check the following hint.");
println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
handleException(e);
return false;
} catch (NumberFormatException e) {
println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
handleException(e);
return false;
}
return true;
}
private static void serve() {
try (Scanner scanner = new Scanner(System.in)) {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
......@@ -93,57 +120,58 @@ public class WinClient extends AbstractClient {
if (password == null) {
password = readPassword();
}
try {
IoTDBConnection connection = (IoTDBConnection) DriverManager.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
System.out.print(IOTDB_CLI_PREFIX + "> ");
String s = scanner.nextLine();
if (s != null) {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length && !isQuit; i++) {
String cmd = cmds[i];
if (cmd != null && !"".equals(cmd.trim())) {
OperationResult result = handleInputCmd(cmd, connection);
switch (result) {
case STOP_OPER:
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
}
if(isQuit) {
break;
}
}
connection.close();
System.out.println(String.format("%s> exit normally.", IOTDB_CLI_PREFIX));
} catch (SQLException e) {
System.out.println(String
.format("%s> %s Host is %s, port is %s.", IOTDB_CLI_PREFIX, e.getMessage(), host,
port));
}
receiveCommands(scanner);
} catch (ArgsErrorException e) {
System.out.println(IOTDB_CLI_PREFIX + "> input params error because" + e.getMessage());
println(IOTDB_CLI_PREFIX + "> input params error because" + e.getMessage());
handleException(e);
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
handleException(e);
}
}
private static String readPassword() {
Console c = System.console();
if (c == null) { // IN ECLIPSE IDE
System.out.print(IOTDB_CLI_PREFIX + "> please input password: ");
Scanner scanner = new Scanner(System.in);
return scanner.nextLine();
} else { // Outside Eclipse IDE
return new String(c.readPassword(IOTDB_CLI_PREFIX + "> please input password: "));
private static void receiveCommands(Scanner scanner) throws TException {
try (IoTDBConnection connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password)) {
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
print(IOTDB_CLI_PREFIX + "> ");
String s = scanner.nextLine();
boolean continues = processCommand(s, connection);
if (!continues) {
break;
}
}
} catch (SQLException e) {
println(String
.format("%s> %s Host is %s, port is %s.", IOTDB_CLI_PREFIX, e.getMessage(), host,
port));
handleException(e);
}
}
private static boolean processCommand(String s, IoTDBConnection connection) {
if (s == null) {
return true;
}
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !"".equals(cmd.trim())) {
OperationResult result = handleInputCmd(cmd, connection);
switch (result) {
case STOP_OPER:
return false;
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
return true;
}
}
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/CsvTestDataGen.java 已删除 100644 → 0
浏览文件 @ a1e3a303
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iotdb.cli.tool;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
@Deprecated
public class CsvTestDataGen {
private CsvTestDataGen() {
}
private static final String PATHS = "Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2."
+ "s1,root.fit.d2.s3";
private static String[] iso = {
PATHS,
"1970-01-01T08:00:00.001+08:00,,1,pass,1,1", "1970-01-01T08:00:00.002+08:00,,2,pass,,",
"1970-01-01T08:00:00.003+08:00,,3,pass,,", "1970-01-01T08:00:00.004+08:00,4,,,4,4"};
private static String[] defaultLong = {
PATHS,
"1,,1,pass,1,1",
"2,,2,pass,,", "1970-01-01T08:00:00.003+08:00,,3,pass,,", "3,4,,,4,4"};
private static String[] userSelfDefine = {
PATHS,
"1971,,1,pass,1,1",
"1972,,2,pass,,", "1973-01-01T08:00:00.003+08:00,,3,pass,,", "1974,4,,,4,4"};
private static FileOutputStream fos = null;
private static OutputStreamWriter osw = null;
private static BufferedWriter bw = null;
private static final String USER_DIR = "user.dir";
/**
* generate iso.csv data.
*
* @return path
*/
public static String isoDataGen() {
String path = System.getProperties().getProperty(USER_DIR) + "/src/test/resources/iso.csv";
File file = new File(path);
writeDataFrom(file, iso);
return path;
}
/**
* generate default long data file: defaultLong.csv .
*
* @return path
*/
public static String defaultLongDataGen() {
String path =
System.getProperties().getProperty(USER_DIR) + "/src/test/resources/defaultLong.csv";
File file = new File(path);
writeDataFrom(file, defaultLong);
return path;
}
/**
* generate user defined data: userSelfDefine.csv .
*
* @return path
*/
public static String userSelfDataGen() {
String path =
System.getProperties().getProperty(USER_DIR) + "/src/test/resources/userSelfDefine.csv";
File file = new File(path);
writeDataFrom(file, userSelfDefine);
return path;
}
private static void writeDataFrom(File file, String[] info) {
try {
if (!file.exists()) {
file.createNewFile();
}
fos = new FileOutputStream(file);
osw = new OutputStreamWriter(fos);
bw = new BufferedWriter(osw);
for (String str : info) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
osw.close();
fos.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static void main(String[] args) {
System.out.println(defaultLongDataGen());
}
}
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
浏览文件 @ 2a472685
......@@ -121,23 +121,22 @@ public class ExportCsv extends AbstractCsvTool {
for (int i = 0; i < values.length; i++) {
dumpResult(values[i], i);
}
return;
} else {
dumpFromSqlFile(sqlFile);
}
} catch (ClassNotFoundException e) {
LOGGER.error(
"Failed to dump data because cannot find TsFile JDBC Driver, "
+ "please check whether you have imported driver or not");
+ "please check whether you have imported driver or not", e);
} catch (SQLException e) {
LOGGER.error("Encounter an error when dumping data, error is {}", e.getMessage());
LOGGER.error("Encounter an error when dumping data, error is ", e);
} catch (IOException e) {
LOGGER.error("Failed to operate on file, because {}", e.getMessage());
LOGGER.error("Failed to operate on file, because ", e);
} catch (TException e) {
LOGGER.error("Encounter an error when connecting to server, because {}",
e.getMessage());
LOGGER.error("Encounter an error when connecting to server, because ",
e);
} catch (ArgsErrorException e) {
e.printStackTrace();
LOGGER.error("Invalid args.", e);
} finally {
reader.close();
if (connection != null) {
......@@ -222,7 +221,7 @@ public class ExportCsv extends AbstractCsvTool {
try {
dumpResult(sql, index);
} catch (SQLException e) {
LOGGER.error("Cannot dump data for statment {}, because {}", sql, e.getMessage());
LOGGER.error("Cannot dump data for statement {}, because ", sql, e);
}
index++;
}
......@@ -238,95 +237,99 @@ public class ExportCsv extends AbstractCsvTool {
*/
private static void dumpResult(String sql, int index)
throws SQLException {
FileWriter fw = null;
BufferedWriter bw = null;
final String path = targetDirectory + DUMP_FILE_NAME + index + ".csv";
File tf = new File(path);
try {
File tf = new File(path);
if (!tf.exists() && !tf.createNewFile()) {
LOGGER.error("Could not create target file for sql statement: {}", sql);
return;
}
fw = new FileWriter(tf);
bw = new BufferedWriter(fw);
} catch (IOException e) {
LOGGER.error(e.getMessage());
return;
}
Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
ResultSetMetaData metadata = rs.getMetaData();
long startTime = System.currentTimeMillis();
try {
try (Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
ResultSetMetaData metadata = rs.getMetaData();
long startTime = System.currentTimeMillis();
int count = metadata.getColumnCount();
// write data in csv file
for (int i = 1; i <= count; i++) {
if (i < count) {
bw.write(metadata.getColumnLabel(i) + ",");
} else {
bw.write(metadata.getColumnLabel(i) + "\n");
}
writeMetadata(bw, count, metadata);
writeResultSet(rs, bw, count);
LOGGER.info("Statement [{}] has dumped to file {} successfully! It costs {}ms.",
sql, path, System.currentTimeMillis() - startTime);
} catch (IOException e) {
LOGGER.error("Cannot dump result because", e);
}
}
private static void writeMetadata(BufferedWriter bw, int count, ResultSetMetaData metadata)
throws SQLException, IOException {
for (int i = 1; i <= count; i++) {
if (i < count) {
bw.write(metadata.getColumnLabel(i) + ",");
} else {
bw.write(metadata.getColumnLabel(i) + "\n");
}
}
}
private static void writeResultSet(ResultSet rs, BufferedWriter bw, int count)
throws SQLException, IOException {
while (rs.next()) {
if (rs.getString(1) == null ||
"null".equalsIgnoreCase(rs.getString(1))) {
bw.write(",");
} else {
writeTime(rs, bw);
writeValue(rs, count, bw);
}
while (rs.next()) {
if (rs.getString(1) == null || "null".equalsIgnoreCase(rs.getString(1))) {
}
}
private static void writeTime(ResultSet rs, BufferedWriter bw) throws SQLException, IOException {
ZonedDateTime dateTime;
switch (timeFormat) {
case DEFAULT_TIME_FORMAT:
case "default":
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
zoneId);
bw.write(dateTime.toString() + ",");
break;
case "timestamp":
case "long":
case "nubmer":
bw.write(rs.getLong(1) + ",");
break;
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) + ",");
break;
}
}
private static void writeValue(ResultSet rs, int count, BufferedWriter bw)
throws SQLException, IOException {
for (int j = 2; j <= count; j++) {
if (j < count) {
if ("null".equals(rs.getString(j))) {
bw.write(",");
} else {
ZonedDateTime dateTime;
switch (timeFormat) {
case DEFAULT_TIME_FORMAT:
case "default":
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
bw.write(dateTime.toString() + ",");
break;
case "timestamp":
case "long":
case "nubmer":
bw.write(rs.getLong(1) + ",");
break;
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) + ",");
break;
}
for (int j = 2; j <= count; j++) {
if (j < count) {
if ("null".equals(rs.getString(j))) {
bw.write(",");
} else {
bw.write(rs.getString(j) + ",");
}
} else {
if ("null".equals(rs.getString(j))) {
bw.write("\n");
} else {
bw.write(rs.getString(j) + "\n");
}
}
}
bw.write(rs.getString(j) + ",");
}
}
LOGGER.info("Statement [{}] has dumped to file {} successfully! It costs {}ms.",
sql, path, System.currentTimeMillis() - startTime);
} catch (IOException e) {
LOGGER.error(e.getMessage());
} finally {
try {
if (rs != null) {
rs.close();
}
if (bw != null) {
bw.close();
}
if (fw != null) {
fw.close();
} else {
if ("null".equals(rs.getString(j))) {
bw.write("\n");
} else {
bw.write(rs.getString(j) + "\n");
}
} catch (IOException e) {
LOGGER.error(e.getMessage());
}
statement.close();
}
}
}
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
浏览文件 @ 2a472685
......@@ -70,8 +70,11 @@ public class ImportCsv extends AbstractCsvTool {
private static final String STRING_DATA_TYPE = "TEXT";
private static final int BATCH_EXECUTE_COUNT = 10;
private static String filename;
private static String errorInsertInfo = "";
private static boolean errorFlag;
private static int count;
private static Statement statement;
/**
* create the commandline options.
......@@ -120,21 +123,21 @@ public class ImportCsv extends AbstractCsvTool {
* Data from csv To tsfile.
*/
private static void loadDataFromCSV(File file, int index) {
Statement statement = null;
FileReader fr = null;
BufferedReader br = null;
FileWriter fw = null;
BufferedWriter bw = null;
statement = null;
File errorFile = new File(errorInsertInfo + index);
boolean errorFlag = true;
try {
fr = new FileReader(file);
br = new BufferedReader(fr);
if (!errorFile.exists()) {
if (!errorFile.exists()) {
try {
errorFile.createNewFile();
} catch (IOException e) {
LOGGER.error("Cannot create a errorFile because, ", e);
return;
}
fw = new FileWriter(errorFile);
bw = new BufferedWriter(fw);
}
errorFlag = true;
try(BufferedReader br = new BufferedReader(new FileReader(file));
BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile))) {
String header = br.readLine();
......@@ -160,112 +163,34 @@ public class ImportCsv extends AbstractCsvTool {
long startTime = System.currentTimeMillis();
Map<String, String> timeseriesDataType = new HashMap<>();
DatabaseMetaData databaseMetaData = connection.getMetaData();
for (int i = 1; i < strHeadInfo.length; i++) {
ResultSet resultSet = databaseMetaData.getColumns(null,
null, strHeadInfo[i], null);
if (resultSet.next()) {
timeseriesDataType.put(resultSet.getString(1),
resultSet.getString(2));
} else {
String errorInfo = String.format("Database cannot find %s in %s, stop import!",
strHeadInfo[i], file.getAbsolutePath());
LOGGER.error("Database cannot find {} in {}, stop import!",
strHeadInfo[i], file.getAbsolutePath());
bw.write(errorInfo);
errorFlag = false;
return;
}
headInfo.add(strHeadInfo[i]);
String deviceInfo = strHeadInfo[i].substring(0, strHeadInfo[i].lastIndexOf('.'));
if (!deviceToColumn.containsKey(deviceInfo)) {
deviceToColumn.put(deviceInfo, new ArrayList<>());
}
// storage every device's sensor index info
deviceToColumn.get(deviceInfo).add(i - 1);
colInfo.add(strHeadInfo[i].substring(strHeadInfo[i].lastIndexOf('.') + 1));
boolean success = queryDatabaseMeta(strHeadInfo, file, bw, timeseriesDataType, headInfo,
deviceToColumn, colInfo);
if (!success) {
errorFlag = false;
return;
}
String line;
statement = connection.createStatement();
int count = 0;
List<String> tmp = new ArrayList<>();
while ((line = br.readLine()) != null) {
List<String> sqls = new ArrayList<>();
try {
sqls = createInsertSQL(line, timeseriesDataType, deviceToColumn, colInfo, headInfo);
} catch (Exception e) {
bw.write(String.format("error input line, maybe it is not complete: %s", line));
bw.newLine();
errorFlag = false;
}
for (String str : sqls) {
try {
count++;
statement.addBatch(str);
tmp.add(str);
if (count == BATCH_EXECUTE_COUNT) {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
}
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
}
}
}
try {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
LOGGER.info("Load data from {} successfully, it takes {}ms", file.getName(),
System.currentTimeMillis() - startTime);
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
success = readAndGenSqls(br, timeseriesDataType, deviceToColumn, colInfo, headInfo,
bw, tmp);
if (!success) {
return;
}
executeSqls(bw, tmp, startTime, file);
} catch (FileNotFoundException e) {
LOGGER.error("Cannot find {}", file.getName());
LOGGER.error("Cannot find {}", file.getName(), e);
} catch (IOException e) {
LOGGER.error("CSV file read exception! {}", e.getMessage());
LOGGER.error("CSV file read exception! ", e);
} catch (SQLException e) {
LOGGER.error("Database connection exception! {}", e.getMessage());
LOGGER.error("Database connection exception!", e);
} finally {
try {
if (fr != null) {
fr.close();
}
if (br != null) {
br.close();
}
if (fw != null) {
fw.close();
}
if (bw != null) {
bw.close();
}
if (statement != null) {
statement.close();
}
......@@ -276,13 +201,132 @@ public class ImportCsv extends AbstractCsvTool {
+ "information", file.getAbsolutePath(), errorFile.getAbsolutePath());
}
} catch (SQLException e) {
System.out.println("[ERROR] Sql statement can not be closed ! " + e.getMessage());
LOGGER.error("Sql statement can not be closed ! ", e);
} catch (IOException e) {
System.out.println("[ERROR] Close file error ! " + e.getMessage());
LOGGER.error("Close file error ! ", e);
}
}
}
private static void executeSqls(BufferedWriter bw, List<String> tmp, long startTime, File file)
throws IOException {
try {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
tmp.clear();
LOGGER.info("Load data from {} successfully, it takes {}ms", file.getName(),
System.currentTimeMillis() - startTime);
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
LOGGER.error("Cannot execute sql because ", e);
}
}
private static boolean readAndGenSqls(BufferedReader br, Map<String, String> timeseriesDataType,
Map<String, ArrayList<Integer>> deviceToColumn, List<String> colInfo,
List<String> headInfo, BufferedWriter bw, List<String> tmp) throws IOException {
String line;
count = 0;
while ((line = br.readLine()) != null) {
List<String> sqls;
try {
sqls = createInsertSQL(line, timeseriesDataType, deviceToColumn, colInfo, headInfo);
} catch (Exception e) {
bw.write(String.format("error input line, maybe it is not complete: %s", line));
bw.newLine();
LOGGER.error("Cannot create sql for {} because ", line, e);
errorFlag = false;
return false;
}
boolean success = addSqlsToBatch(sqls, tmp, bw);
if (!success) {
return false;
}
}
return true;
}
private static boolean addSqlsToBatch(List<String> sqls, List<String> tmp, BufferedWriter bw)
throws IOException {
for (String str : sqls) {
try {
count++;
statement.addBatch(str);
tmp.add(str);
checkBatchSize(bw, tmp);
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
LOGGER.error("Cannot execute sql because ", e);
return false;
}
}
return true;
}
private static void checkBatchSize(BufferedWriter bw, List<String> tmp)
throws SQLException, IOException {
if (count == BATCH_EXECUTE_COUNT) {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
}
}
private static boolean queryDatabaseMeta(String[] strHeadInfo, File file, BufferedWriter bw,
Map<String, String> timeseriesDataType, List<String> headInfo,
Map<String, ArrayList<Integer>> deviceToColumn,
List<String> colInfo)
throws SQLException, IOException {
DatabaseMetaData databaseMetaData = connection.getMetaData();
for (int i = 1; i < strHeadInfo.length; i++) {
ResultSet resultSet = databaseMetaData.getColumns(null,
null, strHeadInfo[i], null);
if (resultSet.next()) {
timeseriesDataType.put(resultSet.getString(1),
resultSet.getString(2));
} else {
String errorInfo = String.format("Database cannot find %s in %s, stop import!",
strHeadInfo[i], file.getAbsolutePath());
LOGGER.error("Database cannot find {} in {}, stop import!",
strHeadInfo[i], file.getAbsolutePath());
bw.write(errorInfo);
return false;
}
headInfo.add(strHeadInfo[i]);
String deviceInfo = strHeadInfo[i].substring(0, strHeadInfo[i].lastIndexOf('.'));
if (!deviceToColumn.containsKey(deviceInfo)) {
deviceToColumn.put(deviceInfo, new ArrayList<>());
}
// storage every device's sensor index info
deviceToColumn.get(deviceInfo).add(i - 1);
colInfo.add(strHeadInfo[i].substring(strHeadInfo[i].lastIndexOf('.') + 1));
}
return true;
}
private static List<String> createInsertSQL(String line, Map<String, String> timeseriesToType,
Map<String, ArrayList<Integer>> deviceToColumn,
List<String> colInfo, List<String> headInfo) {
......@@ -291,42 +335,50 @@ public class ImportCsv extends AbstractCsvTool {
Iterator<Map.Entry<String, ArrayList<Integer>>> it = deviceToColumn.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, ArrayList<Integer>> entry = it.next();
StringBuilder sbd = new StringBuilder();
ArrayList<Integer> colIndex = entry.getValue();
sbd.append("insert into " + entry.getKey() + "(timestamp");
int skipCount = 0;
for (int j = 0; j < colIndex.size(); ++j) {
if ("".equals(data[entry.getValue().get(j) + 1])) {
skipCount++;
continue;
}
sbd.append(", " + colInfo.get(colIndex.get(j)));
String sql = createOneSql(entry, data, colInfo, timeseriesToType, headInfo);
if (sql != null) {
sqls.add(sql);
}
// define every device null value' number, if the number equal the
// sensor number, the insert operation stop
if (skipCount == entry.getValue().size()) {
}
return sqls;
}
private static String createOneSql(Map.Entry<String, ArrayList<Integer>> entry, String[] data,
List<String> colInfo, Map<String, String> timeseriesToType, List<String> headInfo) {
StringBuilder sbd = new StringBuilder();
ArrayList<Integer> colIndex = entry.getValue();
sbd.append("insert into ").append(entry.getKey()).append("(timestamp");
int skipCount = 0;
for (int j = 0; j < colIndex.size(); ++j) {
if ("".equals(data[entry.getValue().get(j) + 1])) {
skipCount++;
continue;
}
sbd.append(", ").append(colInfo.get(colIndex.get(j)));
}
// define every device null value' number, if the number equal the
// sensor number, the insert operation stop
if (skipCount == entry.getValue().size()) {
return null;
}
// TODO when timestampsStr is empty
String timestampsStr = data[0];
sbd.append(") values(").append(timestampsStr.trim().isEmpty()
? "NO TIMESTAMP" : timestampsStr);
// TODO when timestampsStr is empty
String timestampsStr = data[0];
sbd.append(") values(").append(timestampsStr.trim().isEmpty()
? "NO TIMESTAMP" : timestampsStr);
for (int j = 0; j < colIndex.size(); ++j) {
if ("".equals(data[entry.getValue().get(j) + 1])) {
continue;
}
if (timeseriesToType.get(headInfo.get(colIndex.get(j))).equals(STRING_DATA_TYPE)) {
sbd.append(", \'" + data[colIndex.get(j) + 1] + "\'");
} else {
sbd.append("," + data[colIndex.get(j) + 1]);
}
for (int j = 0; j < colIndex.size(); ++j) {
if ("".equals(data[entry.getValue().get(j) + 1])) {
continue;
}
if (timeseriesToType.get(headInfo.get(colIndex.get(j))).equals(STRING_DATA_TYPE)) {
sbd.append(", \'").append(data[colIndex.get(j) + 1]).append("\'");
} else {
sbd.append(",").append(data[colIndex.get(j) + 1]);
}
sbd.append(")");
sqls.add(sbd.toString());
}
return sqls;
sbd.append(")");
return sbd.toString();
}
public static void main(String[] args) throws IOException, SQLException {
......@@ -334,7 +386,7 @@ public class ImportCsv extends AbstractCsvTool {
HelpFormatter hf = new HelpFormatter();
hf.setOptionComparator(null);
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLine commandLine;
CommandLineParser parser = new DefaultParser();
if (args == null || args.length == 0) {
......@@ -345,7 +397,7 @@ public class ImportCsv extends AbstractCsvTool {
try {
commandLine = parser.parse(options, args);
} catch (ParseException e) {
LOGGER.error(e.getMessage());
LOGGER.error("Parse error ", e);
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
......@@ -358,7 +410,7 @@ public class ImportCsv extends AbstractCsvTool {
reader.setExpandEvents(false);
try {
parseBasicParams(commandLine, reader);
filename = commandLine.getOptionValue(FILE_ARGS);
String filename = commandLine.getOptionValue(FILE_ARGS);
if (filename == null) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
......@@ -366,9 +418,9 @@ public class ImportCsv extends AbstractCsvTool {
parseSpecialParams(commandLine);
importCsvFromFile(host, port, username, password, filename, timeZoneID);
} catch (ArgsErrorException e) {
// ignored
LOGGER.error("Args error", e);
} catch (Exception e) {
LOGGER.error("Encounter an error, because {}", e.getMessage());
LOGGER.error("Encounter an error, because ", e);
} finally {
reader.close();
}
......@@ -405,12 +457,12 @@ public class ImportCsv extends AbstractCsvTool {
} catch (ClassNotFoundException e) {
LOGGER.error(
"Failed to dump data because cannot find TsFile JDBC Driver, "
+ "please check whether you have imported driver or not");
+ "please check whether you have imported driver or not", e);
} catch (TException e) {
LOGGER.error("Encounter an error when connecting to server, because {}",
e.getMessage());
LOGGER.error("Encounter an error when connecting to server, because ",
e);
} catch (Exception e) {
LOGGER.error("Encounter an error, because {}", e.getMessage());
LOGGER.error("Encounter an error, because ", e);
} finally {
if (connection != null) {
connection.close();
......@@ -428,7 +480,12 @@ public class ImportCsv extends AbstractCsvTool {
private static void importFromDirectory(File file) {
int i = 1;
for (File subFile : file.listFiles()) {
File[] files = file.listFiles();
if (files == null) {
return;
}
for (File subFile : files) {
if (subFile.isFile()) {
if (subFile.getName().endsWith(FILE_SUFFIX)) {
loadDataFromCSV(subFile, i);
......
iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
浏览文件 @ 2a472685
......@@ -271,8 +271,6 @@ public class DeletionQueryTest {
FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
FileNodeManager.getInstance().forceFlush(UsageLevel.DANGEROUS);
Thread.sleep(3000);
FileNodeManager.getInstance().closeAll();
List<Path> pathList = new ArrayList<>();
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或