From 01062e669192551ea62d5b0dc408495d491146b2 Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 14 Aug 2019 11:16:22 +0800 Subject: [PATCH] upload code for cassandra/tdengine test --- .../comparisonTest/cassandra/application.conf | 5 + .../cassandra/cassandratest/pom.xml | 131 ++++++++++++ .../com/cassandra/test/CassandraTest.java | 200 ++++++++++++++++++ .../java/com/cassandra/test/WriteThread.java | 99 +++++++++ tests/comparisonTest/cassandra/cqlstr | 16 ++ tests/comparisonTest/cassandra/q1.txt | 11 + tests/comparisonTest/cassandra/q2.txt | 50 +++++ tests/comparisonTest/cassandra/q3.txt | 10 + tests/comparisonTest/cassandra/q4.txt | 10 + 9 files changed, 532 insertions(+) create mode 100644 tests/comparisonTest/cassandra/application.conf create mode 100644 tests/comparisonTest/cassandra/cassandratest/pom.xml create mode 100644 tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java create mode 100644 tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java create mode 100644 tests/comparisonTest/cassandra/cqlstr create mode 100644 tests/comparisonTest/cassandra/q1.txt create mode 100644 tests/comparisonTest/cassandra/q2.txt create mode 100644 tests/comparisonTest/cassandra/q3.txt create mode 100644 tests/comparisonTest/cassandra/q4.txt diff --git a/tests/comparisonTest/cassandra/application.conf b/tests/comparisonTest/cassandra/application.conf new file mode 100644 index 0000000000..23b1068209 --- /dev/null +++ b/tests/comparisonTest/cassandra/application.conf @@ -0,0 +1,5 @@ +datastax-java-driver { + basic.request { + timeout = 200000 seconds + } +} diff --git a/tests/comparisonTest/cassandra/cassandratest/pom.xml b/tests/comparisonTest/cassandra/cassandratest/pom.xml new file mode 100644 index 0000000000..01667ff53a --- /dev/null +++ b/tests/comparisonTest/cassandra/cassandratest/pom.xml @@ -0,0 +1,131 @@ + + + + 4.0.0 + + com.cassandra.test + cassandratest + 1.0-SNAPSHOT + jar + + + + + org.apache.maven.plugins + maven-plugins + 30 + + + org.apache.maven.plugins + maven-assembly-plugin + 3.0.0 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + CassandraTest + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 8 + 8 + + + + + + + cassandratest + + http://www.example.com + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 4.11 + test + + + + com.datastax.oss + java-driver-core + 4.1.0 + + + + com.datastax.oss + java-driver-query-builder + 4.1.0 + + + + com.datastax.oss + java-driver-mapper-runtime + 4.1.0 + + + + commons-io + commons-io + 2.4 + + + + org.apache.commons + commons-lang3 + 3.7 + + + + org.slf4j + slf4j-api + 1.7.5 + + + org.slf4j + slf4j-log4j12 + 1.7.5 + + + + org.apache.logging.log4j + log4j-1.2-api + 2.8.2 + + + + + diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java b/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java new file mode 100644 index 0000000000..a3f72de13c --- /dev/null +++ b/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java @@ -0,0 +1,200 @@ +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.*; +import com.datastax.oss.driver.api.core.session.*; +import com.datastax.oss.driver.api.core.config.*; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +//import com.datastax.driver.core.Cluster; +//import com.datastax.driver.core.Cluster; + +import java.io.BufferedWriter; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.FileReader; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.Random; +import java.math.*; +import java.lang.reflect.Method; + + +public class CassandraTest{ + + public static void main(String args[]) { + + + // begin to parse argument + String datadir = "/home/ubuntu/testdata"; + String sqlfile = "/home/ubuntu/fang/cassandra/q1.txt"; + String cfgfile = "/home/ubuntu/fang/cassandra/application.conf"; + boolean q4flag = false; + int numOfRows = 1000000; + int numOfFiles =0; + int numOfClients =0; + int rowsPerRequest =0; + for (int i = 0; i < args.length; ++i) { + if (args[i].equalsIgnoreCase("-dataDir")) { + if (i < args.length - 1) { + datadir = args[++i]; + } + } else if (args[i].equalsIgnoreCase("-numofFiles")) { + if (i < args.length - 1) { + numOfFiles = Integer.parseInt(args[++i]); + } + } else if (args[i].equalsIgnoreCase("-rowsPerRequest")) { + if (i < args.length - 1) { + rowsPerRequest = Integer.parseInt(args[++i]); + } + } else if (args[i].equalsIgnoreCase("-writeClients")) { + if (i < args.length - 1) { + numOfClients = Integer.parseInt(args[++i]); + } + } else if (args[i].equalsIgnoreCase("-sql")) { + sqlfile = args[++i]; + } else if (args[i].equalsIgnoreCase("-timetest")) { + q4flag = true; + } else if (args[i].equalsIgnoreCase("-conf")) { + cfgfile = args[++i]; + } + } + // file below to make sure no timeout error + File confile = new File(cfgfile); + + System.out.println("parameters\n"); + + if (numOfFiles >0) { + // write data + System.out.printf("----dataDir:%s\n", datadir); + System.out.printf("----numOfFiles:%d\n", numOfFiles); + System.out.printf("----numOfClients:%d\n", numOfClients); + System.out.printf("----rowsPerRequest:%d\n", rowsPerRequest); + + // connect to cassandra server + System.out.printf("----connecting to cassandra server\n"); + try { + CqlSession session = CqlSession.builder() + .withConfigLoader(DriverConfigLoader.fromFile(confile)) + .build(); + + session.execute("drop keyspace if exists cassandra"); + session.execute("CREATE KEYSPACE if not exists cassandra WITH replication = {'class':'SimpleStrategy', 'replication_factor':1}"); + if (q4flag) { + session.execute("create table if not exists cassandra.test (devid int, devname text, devgroup int, ts bigint, minute bigint, temperature int, humidity float ,primary key (minute,ts,devgroup,devid,devname))"); + } else { + session.execute("create table if not exists cassandra.test (devid int, devname text, devgroup int, ts bigint, temperature int, humidity float ,primary key (devgroup,devid,devname,ts))"); + } + session.close(); + System.out.printf("----created keyspace cassandra and table test\n"); + + // begin to insert data + System.out.printf("----begin to insert data\n"); + long startTime = System.currentTimeMillis(); + int a = numOfFiles/numOfClients; + int b = numOfFiles%numOfClients; + int last = 0; + + WriteThread[] writethreads = new WriteThread[numOfClients]; + int[] wargs = new int[2]; // data file start, end + wargs[0] = numOfRows; //rows to be read from each file + wargs[1] = rowsPerRequest; + int fstart =0; + int fend =0; + for (int i = 0; i10) { + long startTime = System.currentTimeMillis(); + // begin to query one line command // + // end querying one line command + try { + + ResultSet results = session.execute(line); + long icounter = 0; + for (Row row : results) { + icounter++; + } + + long stopTime = System.currentTimeMillis(); + float elapseTime = stopTime - startTime; + elapseTime = elapseTime/1000; + System.out.printf("----spend %f seconds to query: %s\n", elapseTime, line); + } catch (Exception ex) { + ex.printStackTrace(); + System.out.printf("---- query failed!\n"); + System.exit(1); + } + + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + session.close(); + } + } catch (Exception ex) { + ex.printStackTrace(); + } finally { + System.out.println("query end:----\n"); + } + } // end write or query + System.exit(0); + }// end main +}// end class diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java b/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java new file mode 100644 index 0000000000..7d8679a303 --- /dev/null +++ b/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java @@ -0,0 +1,99 @@ +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.math.*; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.*; +import com.datastax.oss.driver.api.core.session.*; +import com.datastax.oss.driver.api.core.config.*; + + +public class WriteThread extends Thread { + + private int[] wargs; // fstart, fend, rows to be read, rows perrequest + private String fdir; + private int fstart; + private int fend; + private boolean q4flag; + + public WriteThread (int fstart, int fend,int[] wargs, String fdir, boolean q4flag) { + this.fstart = fstart; + this.fend = fend; + this.fdir = fdir; + this.wargs = wargs; + this.q4flag = q4flag; + } + + // begin to insert in this thread + public void run() { + /* + // this configuration file makes sure no timeout error + File confile = new File("/home/ubuntu/fang/cassandra/application.conf"); + */ + // connect to server + try { + CqlSession session = CqlSession.builder() + //.withConfigLoader(DriverConfigLoader.fromFile(confile)) + .build(); + //session.execute("use cassandra"); + int tominute = 6000; + for (int i=fstart; i<=fend; i++) { + String csvfile; + csvfile = fdir + "/testdata"+ Integer.toString(i)+".csv"; + BufferedReader br = null; + String line = ""; + String cvsSplitBy = " "; + try { + br = new BufferedReader(new FileReader(csvfile)); + System.out.println("---- begin to read file " +csvfile+"\n"); + for (int itotalrow =0; itotalrow