From 2a209718d7d4d15f7edc5b84acae105e590435ca Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sdsang@taosdata.com>
Date: Mon, 8 Jun 2020 08:59:53 +0000
Subject: [PATCH] add regression test suite

[TD-551]
---
 tests/pytest/regressiontest.sh   | 136 ++++++++++++++++++++
 tests/script/regressionSuite.sim | 213 +++++++++++++++++++++++++++++++
 tests/test-all.sh                |  14 +-
 3 files changed, 361 insertions(+), 2 deletions(-)
 create mode 100755 tests/pytest/regressiontest.sh
 create mode 100644 tests/script/regressionSuite.sim

diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh
new file mode 100755
index 0000000000..e1d4c6348b
--- /dev/null
+++ b/tests/pytest/regressiontest.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+ulimit -c unlimited
+
+python3 ./test.py -f insert/basic.py
+python3 ./test.py -f insert/int.py
+python3 ./test.py -f insert/float.py
+python3 ./test.py -f insert/bigint.py
+python3 ./test.py -f insert/bool.py
+python3 ./test.py -f insert/double.py
+python3 ./test.py -f insert/smallint.py
+python3 ./test.py -f insert/tinyint.py
+python3 ./test.py -f insert/date.py
+python3 ./test.py -f insert/binary.py
+python3 ./test.py -f insert/nchar.py
+# python3 ./test.py -f insert/nchar-boundary.py
+# python3 ./test.py -f insert/nchar-unicode.py
+python3 ./test.py -f insert/multi.py
+python3 ./test.py -f insert/randomNullCommit.py
+
+python3 ./test.py -f table/column_name.py
+python3 ./test.py -f table/column_num.py
+python3 ./test.py -f table/db_table.py
+# python3 ./test.py -f table/tablename-boundary.py
+
+# tag
+python3 ./test.py -f tag_lite/filter.py
+python3 ./test.py -f tag_lite/create-tags-boundary.py
+python3 ./test.py -f tag_lite/3.py
+python3 ./test.py -f tag_lite/4.py
+python3 ./test.py -f tag_lite/5.py
+python3 ./test.py -f tag_lite/6.py
+# python3 ./test.py -f tag_lite/add.py
+python3 ./test.py -f tag_lite/bigint.py
+python3 ./test.py -f tag_lite/binary_binary.py
+python3 ./test.py -f tag_lite/binary.py
+python3 ./test.py -f tag_lite/bool_binary.py
+python3 ./test.py -f tag_lite/bool_int.py
+python3 ./test.py -f tag_lite/bool.py
+python3 ./test.py -f tag_lite/change.py
+python3 ./test.py -f tag_lite/column.py
+# python3 ./test.py -f tag_lite/commit.py
+python3 ./test.py -f tag_lite/create.py
+# python3 ./test.py -f tag_lite/datatype.py
+python3 ./test.py -f tag_lite/datatype-without-alter.py
+# python3 ./test.py -f tag_lite/delete.py
+python3 ./test.py -f tag_lite/double.py
+python3 ./test.py -f tag_lite/float.py
+python3 ./test.py -f tag_lite/int_binary.py
+python3 ./test.py -f tag_lite/int_float.py
+python3 ./test.py -f tag_lite/int.py
+# python3 ./test.py -f tag_lite/set.py
+python3 ./test.py -f tag_lite/smallint.py
+python3 ./test.py -f tag_lite/tinyint.py
+
+# python3 ./test.py -f dbmgmt/database-name-boundary.py
+
+python3 ./test.py -f import_merge/importBlock1HO.py
+python3 ./test.py -f import_merge/importBlock1HPO.py
+python3 ./test.py -f import_merge/importBlock1H.py
+python3 ./test.py -f import_merge/importBlock1S.py
+python3 ./test.py -f import_merge/importBlock1Sub.py
+python3 ./test.py -f import_merge/importBlock1TO.py
+python3 ./test.py -f import_merge/importBlock1TPO.py
+python3 ./test.py -f import_merge/importBlock1T.py
+python3 ./test.py -f import_merge/importBlock2HO.py
+python3 ./test.py -f import_merge/importBlock2HPO.py
+python3 ./test.py -f import_merge/importBlock2H.py
+python3 ./test.py -f import_merge/importBlock2S.py
+python3 ./test.py -f import_merge/importBlock2Sub.py
+python3 ./test.py -f import_merge/importBlock2TO.py
+python3 ./test.py -f import_merge/importBlock2TPO.py
+python3 ./test.py -f import_merge/importBlock2T.py
+python3 ./test.py -f import_merge/importBlockbetween.py
+python3 ./test.py -f import_merge/importCacheFileHO.py
+python3 ./test.py -f import_merge/importCacheFileHPO.py
+python3 ./test.py -f import_merge/importCacheFileH.py
+python3 ./test.py -f import_merge/importCacheFileS.py
+python3 ./test.py -f import_merge/importCacheFileSub.py
+python3 ./test.py -f import_merge/importCacheFileTO.py
+python3 ./test.py -f import_merge/importCacheFileTPO.py
+python3 ./test.py -f import_merge/importCacheFileT.py
+python3 ./test.py -f import_merge/importDataH2.py
+# python3 ./test.py -f import_merge/importDataHO2.py
+# python3 ./test.py -f import_merge/importDataHO.py
+python3 ./test.py -f import_merge/importDataHPO.py
+python3 ./test.py -f import_merge/importDataLastHO.py
+python3 ./test.py -f import_merge/importDataLastHPO.py
+python3 ./test.py -f import_merge/importDataLastH.py
+python3 ./test.py -f import_merge/importDataLastS.py
+python3 ./test.py -f import_merge/importDataLastSub.py
+python3 ./test.py -f import_merge/importDataLastTO.py
+python3 ./test.py -f import_merge/importDataLastTPO.py
+python3 ./test.py -f import_merge/importDataLastT.py
+python3 ./test.py -f import_merge/importDataS.py
+# python3 ./test.py -f import_merge/importDataSub.py
+python3 ./test.py -f import_merge/importDataTO.py
+python3 ./test.py -f import_merge/importDataTPO.py
+python3 ./test.py -f import_merge/importDataT.py
+python3 ./test.py -f import_merge/importHeadOverlap.py
+python3 ./test.py -f import_merge/importHeadPartOverlap.py
+python3 ./test.py -f import_merge/importHead.py
+python3 ./test.py -f import_merge/importHORestart.py
+python3 ./test.py -f import_merge/importHPORestart.py
+python3 ./test.py -f import_merge/importHRestart.py
+python3 ./test.py -f import_merge/importLastHO.py
+python3 ./test.py -f import_merge/importLastHPO.py
+python3 ./test.py -f import_merge/importLastH.py
+python3 ./test.py -f import_merge/importLastS.py
+python3 ./test.py -f import_merge/importLastSub.py
+python3 ./test.py -f import_merge/importLastTO.py
+python3 ./test.py -f import_merge/importLastTPO.py
+python3 ./test.py -f import_merge/importLastT.py
+python3 ./test.py -f import_merge/importSpan.py
+python3 ./test.py -f import_merge/importSRestart.py
+python3 ./test.py -f import_merge/importSubRestart.py
+python3 ./test.py -f import_merge/importTailOverlap.py
+python3 ./test.py -f import_merge/importTailPartOverlap.py
+python3 ./test.py -f import_merge/importTail.py
+python3 ./test.py -f import_merge/importToCommit.py
+python3 ./test.py -f import_merge/importTORestart.py
+python3 ./test.py -f import_merge/importTPORestart.py
+python3 ./test.py -f import_merge/importTRestart.py
+python3 ./test.py -f import_merge/importInsertThenImport.py
+
+# user
+python3 ./test.py -f user/user_create.py
+python3 ./test.py -f user/pass_len.py
+
+# table
+#python3 ./test.py -f table/del_stable.py
+
+#query
+python3 ./test.py -f query/filter.py
+# python3 ./test.py -f query/filterCombo.py
+# python3 ./test.py -f query/queryNormal.py
+# python3 ./test.py -f query/queryError.py
diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim
new file mode 100644
index 0000000000..67f54523df
--- /dev/null
+++ b/tests/script/regressionSuite.sim
@@ -0,0 +1,213 @@
+#unsupport run general/alter/cached_schema_after_alter.sim
+#unsupport run general/alter/count.sim
+#unsupport run general/alter/import.sim
+#unsupport run general/alter/insert1.sim
+#unsupport run general/alter/insert2.sim
+#unsupport run general/alter/metrics.sim
+#unsupport run general/alter/table.sim
+run general/cache/new_metrics.sim
+run general/cache/restart_metrics.sim
+run general/cache/restart_table.sim
+run general/connection/connection.sim  
+run general/column/commit.sim
+run general/column/metrics.sim
+run general/column/table.sim
+run general/compress/commitlog.sim
+run general/compress/compress.sim
+run general/compress/compress2.sim
+run general/compress/uncompress.sim
+run general/compute/avg.sim
+run general/compute/bottom.sim
+run general/compute/count.sim
+run general/compute/diff.sim
+run general/compute/diff2.sim
+run general/compute/first.sim
+run general/compute/interval.sim
+run general/compute/last.sim
+run general/compute/leastsquare.sim
+run general/compute/max.sim
+run general/compute/min.sim
+run general/compute/null.sim
+run general/compute/percentile.sim
+run general/compute/stddev.sim
+run general/compute/sum.sim
+run general/compute/top.sim
+run general/db/alter_option.sim
+run general/db/alter_tables_d2.sim
+run general/db/alter_tables_v1.sim
+run general/db/alter_tables_v4.sim
+run general/db/alter_vgroups.sim
+run general/db/basic.sim
+run general/db/basic1.sim
+run general/db/basic2.sim
+run general/db/basic3.sim
+run general/db/basic4.sim
+run general/db/basic5.sim
+run general/db/delete_reuse1.sim
+run general/db/delete_reuse2.sim
+run general/db/delete_reusevnode.sim
+run general/db/delete_reusevnode2.sim
+run general/db/delete_writing1.sim
+run general/db/delete_writing2.sim
+run general/db/delete.sim
+run general/db/len.sim
+run general/db/repeat.sim
+run general/db/tables.sim
+run general/db/vnodes.sim
+run general/field/2.sim
+run general/field/3.sim
+run general/field/4.sim
+run general/field/5.sim
+run general/field/6.sim
+run general/field/bigint.sim
+run general/field/binary.sim
+run general/field/bool.sim
+run general/field/single.sim
+run general/field/smallint.sim
+run general/field/tinyint.sim
+run general/http/restful.sim
+run general/http/restful_insert.sim
+run general/http/restful_limit.sim
+run general/http/restful_full.sim
+run general/http/prepare.sim
+run general/http/telegraf.sim
+# run general/http/grafana_bug.sim
+# run general/http/grafana.sim
+run general/import/basic.sim
+run general/import/commit.sim
+run general/import/large.sim
+run general/import/replica1.sim
+run general/insert/basic.sim
+run general/insert/insert_drop.sim
+run general/insert/query_block1_memory.sim
+run general/insert/query_block2_memory.sim
+run general/insert/query_block1_file.sim
+run general/insert/query_block2_file.sim
+run general/insert/query_file_memory.sim
+run general/insert/query_multi_file.sim
+run general/insert/tcp.sim
+#unsupport run general/parser/alter.sim
+#unsupport run general/parser/alter1.sim
+#unsupport run general/parser/alter_stable.sim
+run general/parser/auto_create_tb.sim
+run general/parser/auto_create_tb_drop_tb.sim
+run general/parser/col_arithmetic_operation.sim
+run general/parser/columnValue.sim
+# run general/parser/commit.sim
+run general/parser/create_db.sim
+run general/parser/create_mt.sim
+run general/parser/create_tb.sim
+run general/parser/dbtbnameValidate.sim
+run general/parser/import_commit1.sim
+run general/parser/import_commit2.sim
+run general/parser/import_commit3.sim
+run general/parser/insert_tb.sim
+# run general/parser/first_last.sim
+#unsupport run general/parser/import_file.sim
+# run general/parser/lastrow.sim
+run general/parser/nchar.sim
+#unsupport run general/parser/null_char.sim
+# run general/parser/single_row_in_tb.sim
+run general/parser/select_from_cache_disk.sim
+# run general/parser/limit.sim
+# run general/parser/limit1.sim
+# run general/parser/limit1_tblocks100.sim
+# run general/parser/mixed_blocks.sim
+# run general/parser/selectResNum.sim
+run general/parser/select_across_vnodes.sim
+run general/parser/slimit1.sim
+run general/parser/tbnameIn.sim
+run general/parser/binary_escapeCharacter.sim
+# run general/parser/projection_limit_offset.sim
+run general/parser/limit2.sim
+# run general/parser/slimit.sim
+run general/parser/fill.sim
+# run general/parser/fill_stb.sim
+# run general/parser/interp.sim
+# run general/parser/where.sim
+#unsupport run general/parser/join.sim
+#unsupport run general/parser/join_multivnode.sim
+# run general/parser/select_with_tags.sim
+#unsupport run general/parser/groupby.sim
+#unsupport run general/parser/bug.sim
+#unsupport run general/parser/tags_dynamically_specifiy.sim
+#unsupport run general/parser/set_tag_vals.sim
+#unsupport run general/parser/repeatAlter.sim
+#unsupport run general/parser/slimit_alter_tags.sim
+#unsupport run general/parser/stream_on_sys.sim
+#unsupport run general/parser/stream.sim
+#unsupport run general/parser/repeatStream.sim
+run general/stable/disk.sim
+run general/stable/dnode3.sim
+run general/stable/metrics.sim
+run general/stable/values.sim
+run general/stable/vnode3.sim
+# run general/table/autocreate.sim
+run general/table/basic1.sim
+run general/table/basic2.sim
+run general/table/basic3.sim
+run general/table/bigint.sim
+run general/table/binary.sim
+run general/table/bool.sim
+run general/table/column_name.sim
+run general/table/column_num.sim
+run general/table/column_value.sim
+run general/table/column2.sim
+run general/table/date.sim
+run general/table/db.table.sim
+run general/table/delete_reuse1.sim
+run general/table/delete_reuse2.sim
+run general/table/delete_writing.sim
+run general/table/describe.sim
+run general/table/double.sim
+run general/table/fill.sim
+run general/table/float.sim
+run general/table/int.sim
+run general/table/limit.sim
+run general/table/smallint.sim
+run general/table/table_len.sim
+# run general/table/table.sim
+run general/table/tinyint.sim
+run general/table/vgroup.sim
+run general/tag/3.sim
+run general/tag/4.sim
+run general/tag/5.sim
+run general/tag/6.sim
+#unsupport run general/tag/add.sim
+run general/tag/bigint.sim
+run general/tag/binary_binary.sim
+run general/tag/binary.sim
+run general/tag/bool_binary.sim
+run general/tag/bool_int.sim
+run general/tag/bool.sim
+#unsupport run general/tag/change.sim
+run general/tag/column.sim
+#unsupport run general/tag/commit.sim
+run general/tag/create.sim
+#unsupport run general/tag/delete.sim
+run general/tag/double.sim
+run general/tag/filter.sim
+run general/tag/float.sim
+run general/tag/int_binary.sim
+run general/tag/int_float.sim
+run general/tag/int.sim
+#unsupport run general/tag/set.sim
+run general/tag/smallint.sim
+run general/tag/tinyint.sim
+run general/user/authority.sim
+run general/user/monitor.sim
+run general/user/pass_alter.sim
+run general/user/pass_len.sim
+run general/user/user_create.sim
+run general/user/user_len.sim
+run general/vector/metrics_field.sim
+run general/vector/metrics_mix.sim
+run general/vector/metrics_query.sim
+run general/vector/metrics_tag.sim
+run general/vector/metrics_time.sim
+run general/vector/multi.sim
+run general/vector/single.sim
+run general/vector/table_field.sim
+run general/vector/table_mix.sim
+run general/vector/table_query.sim
+run general/vector/table_time.sim
diff --git a/tests/test-all.sh b/tests/test-all.sh
index cd5444858e..cee638e03c 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -24,14 +24,19 @@ GREEN_DARK='\033[0;32m'
 GREEN_UNDERLINE='\033[4;32m'
 NC='\033[0m'
 
-echo "### run TSIM script ###"
+echo "### run TSIM test case ###"
 cd script
 
 [ -f out.log ] && rm -f out.log
 
 if [ "$1" == "cron" ]; then
+  echo "### run TSIM regression test ###"
+  runSimCaseOneByOne regressionSuite.sim
+elif [ "$1" == "full" ]; then
+  echo "### run TSIM full test ###"
   runSimCaseOneByOne fullGeneralSuite.sim
 else
+  echo "### run TSIM smoke test ###"
   runSimCaseOneByOne basicSuite.sim
 fi
 
@@ -53,14 +58,19 @@ if [ "$totalFailed" -ne "0" ]; then
 #  exit $totalFailed
 fi
 
-echo "### run Python script ###"
+echo "### run Python test case ###"
 cd ../pytest
 
 [ -f pytest-out.log ] && rm -f pytest-out.log
 
 if [ "$1" == "cron" ]; then
+  echo "### run Python regression test ###"
+  runPyCaseOneByOne regressiontest.sh
+elif [ "$1" == "full" ]; then
+  echo "### run Python full test ###"
   runPyCaseOneByOne fulltest.sh
 else
+  echo "### run Python smoke test ###"
   runPyCaseOneByOne smoketest.sh
 fi
 totalPySuccess=`grep 'successfully executed' pytest-out.log | wc -l`
-- 
GitLab