Merge pull request #1618 from taosdata/feature/mpeer

Feature/mpeer

.travis.yml

 #
 # Configuration
 #
-git:
-  depth: 1
+#
+# Build Matrix
+#
+matrix:
+  - os: linux
+    language: c
 
-language: c
+    git:
+      - depth: 1
 
-compiler:
-  - clang
-  - gcc
+    compiler: gcc
+    env: DESC="linux/gcc build and test"
 
-os:
-  - linux
-#  - osx
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+          - net-tools
+          - python-pip
+          - python-setuptools
+          - python3-pip
+          - python3-setuptools
+
+    before_install:
+      - sudo apt update -y -qq
+      - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
 
-before_install:
+    script:
+      - cmake ..
+      - make
+
+    after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
-        sudo apt -y update
-        sudo apt -y install python-pip python3-pip python-setuptools python3-setuptools
+          cd ${TRAVIS_BUILD_DIR}/debug
+          sudo make install || exit $?
+
+          pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
+
+          cd ${TRAVIS_BUILD_DIR}/tests
+          bash ./test-all.sh
+
+          if [ "$?" -ne "0" ]; then
+            exit $?
+          fi
+
           ;;
         esac
 
-addons:
+  - os: linux
+    language: c
+    compiler: gcc
+    env: COVERITY_SCAN=true
+    git:
+      - depth: 1
+
+    script:
+      - echo "this job is for coverity scan"
+
+    addons:
       coverity_scan:
 
         # GitHub project metadata
         # ** specific to your project **
         project:
-      name: TDengine
+          name: sangshuduo/TDengine
           version: 2.x
-      description: TDengine
+          description: sangshuduo/TDengine
 
         # Where email notification of build analysis results will be sent
-    notification_email: sdsang@taosdata.com
+        notification_email: sangshuduo@gmail.com
 
         # Commands to prepare for build_command
         # ** likely specific to your build **
@@ -49,79 +94,124 @@ addons:
         #   https://scan.coverity.com/faq#frequency
         branch_pattern: coverity_scan
 
-before_script:
+  - os: linux
+    language: c
+    compiler: gcc
+    env: ENV_COVER=true
+
+    git:
+      - depth: 1
+
+    addons:
+      apt:
+        packages:
+          - build-essential
+          - cmake
+          - net-tools
+          - python-pip
+          - python-setuptools
+          - python3-pip
+          - python3-setuptools
+          - lcov
+
+    before_install:
+      - sudo apt update -y -qq
+      - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools lcov
+
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
       - mkdir debug
       - cd debug
 
-script:
-  - cmake .. 
-  - cmake --build . || exit $?
+    script:
+      - cmake -DCOVER=true ..
+      - make
+
+    after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
-        # Color setting
-        RED='\033[0;31m'
-        GREEN='\033[1;32m'
-        GREEN_DARK='\033[0;32m'
-        GREEN_UNDERLINE='\033[4;32m'
-        NC='\033[0m'
+          cd ${TRAVIS_BUILD_DIR}/debug
+          sudo make install || exit $?
 
-        sudo make install
+          pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
-        cd ../tests/script
-        sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt
+          cd ${TRAVIS_BUILD_DIR}/tests
 
-        total_success=`grep success out.txt | wc -l`
+          bash ./test-all.sh
 
-        if [ "$total_success" -gt "0" ]; then
-          total_success=`expr $total_success - 1`
-          echo -e "${GREEN} ### Total $total_success TSIM case(s) succeed! ### ${NC}"
+          if [ "$?" -ne "0" ]; then
+            exit $?
           fi
 
-        total_failed=`grep failed out.txt | wc -l`
-        if [ "$total_failed" -ne "0" ]; then
-          echo -e "${RED} ### Total $total_failed TSIM case(s) failed! ### ${NC}"
-          exit $total_failed
-        fi
+          cd ${TRAVIS_BUILD_DIR}
+          lcov -d . --capture -o coverage.info
+          lcov -l coverage.info || exit $?
 
-        pip install --user ../../src/connector/python/linux/python2/ 
-        pip3 install --user ../../src/connector/python/linux/python3/ 
+          gem install coveralls-lcov
 
-        cd ../pytest
-        sudo ./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt
-        total_py_success=`grep 'successfully executed' pytest-out.txt | wc -l`
+          # Color setting
+          RED='\033[0;31m'
+          GREEN='\033[1;32m'
+          GREEN_DARK='\033[0;32m'
+          GREEN_UNDERLINE='\033[4;32m'
+          NC='\033[0m'
 
-        if [ "$total_py_success" -gt "0" ]; then
-          echo -e "${GREEN} ### Total $total_py_success python case(s) succeed! ### ${NC}"
+          coveralls-lcov coverage.info
+          if [ "$?" -eq "0" ]; then
+            echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}"
+          else
+            echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} "
+            exit $?
           fi
 
-        total_py_failed=`grep 'failed' pytest-out.txt | wc -l`
-        if [ "$total_py_failed" -ne "0" ]; then
-          echo -e "${RED} ### Total $total_py_failed python case(s) failed! ### ${NC}"
-          exit $total_py_failed
+          bash <(curl -s https://codecov.io/bash) -f coverage.info
+          if [ "$?" -eq "0" ]; then
+            echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
+          else
+            echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} "
+            exit $?
           fi
 
           ;;
         esac
 
-#
-# Build Matrix
-#
-matrix:
   - os: linux
+    language: c
+    compiler: clang
+    env: DESC="linux/clang build"
+    git:
+      - depth: 1
+
     addons:
       apt:
         packages:
           - build-essential
           - cmake
-          - net-tools
-          - python-pip
-          - python-setuptools
-          - python3-pip
-          - python3-setuptools
 
-#  - os: osx
-#    addons:
-#      homebrew:
-#        - cmake
+    before_script:
+      - cd ${TRAVIS_BUILD_DIR}
+      - mkdir debug
+      - cd debug
 
+    script:
+      - cmake ..
+      - make
+
+        #  - os: osx
+        #    language: c
+        #    compiler: clang
+        #    env: DESC="mac/clang build"
+        #    git:
+        #      - depth: 1
+        #    addons:
+        #      homebrew:
+        #        - cmake
+        #
+        #    script:
+        #      - cd ${TRAVIS_BUILD_DIR}
+        #      - mkdir debug
+        #      - cd debug
+        #      - cmake ..
+        #      - make

cmake/platform.inc

@@ -122,6 +122,12 @@ IF (TD_LINUX_64)
       link_libraries(/usr/lib/libargp.a)
       ADD_DEFINITIONS(-D_ALPINE)
     ENDIF ()
+    IF (${COVER} MATCHES "true")
+      MESSAGE(STATUS "Test coverage mode, add extra flags")
+      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+    ENDIF ()
   ELSEIF (TD_LINUX_32)
     IF (NOT TD_ARM)
       EXIT ()
@@ -138,6 +144,12 @@ IF (TD_LINUX_64)
       link_library(/usr/lib/libargp.a)
       ADD_DEFINITIONS(-D_ALPINE)
     ENDIF ()
+    IF (${COVER} MATCHES "true")
+      MESSAGE(STATUS "Test coverage mode, add extra flags")
+      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+    ENDIF ()
   ELSEIF (TD_WINDOWS_64)
     SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
     IF (NOT TD_GODLL)

tests/pytest/insert/basic.py

@@ -41,8 +41,8 @@ class TDTestCase:
 #        tdSql.execute('insert into tb values (now - 7m , 10)')
 #        tdSql.execute('insert into tb values (now - 8m , 10)')
 
-#        tdSql.query("select * from tb")
-#        tdSql.checkRows(insertRows)
+        tdSql.query("select * from tb")
+        tdSql.checkRows(insertRows)
 
     def stop(self):
         tdSql.close()

tests/pytest/simpletest.sh

-sudo python3 ./test.py -f insert/basic.py
+sudo python ./test.py -f insert/basic.py

tests/test/c/CMakeLists.txt

@@ -11,4 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
 
   add_executable(insertPerRow insertPerRow.c)
   target_link_libraries(insertPerRow taos_static pthread)
+
+  add_executable(importOneRow importOneRow.c)
+  target_link_libraries(importOneRow taos_static pthread)
 ENDIF()

tests/test/c/importOneRow.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tlog.h"
+#include "ttimer.h"
+#include "tutil.h"
+
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+void taos_error(TAOS *taos);
+void* taos_execute(void *param);
+
+typedef struct {
+  pthread_t pid;
+  int       index;
+} ThreadObj;
+
+int threadNum = 1;
+int rowNum = 1000;
+int replica = 1;
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance of TDengine\n After writing one row of data to all tables, write the next row\n");
+
+  printf("%s%s\n", indent, "-r");
+  printf("%s%s%s%d\n", indent, indent, "Number of records to write table, default is ", rowNum);
+  printf("%s%s\n", indent, "-t");
+  printf("%s%s%s%d\n", indent, indent, "Number of threads to be used, default is ", threadNum);
+  printf("%s%s\n", indent, "-replica");
+  printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica);
+
+  exit(EXIT_SUCCESS);
+}
+
+void shellParseArgument(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-r") == 0) {
+      rowNum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t") == 0) {
+      threadNum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-replica") == 0) {
+      replica = atoi(argv[++i]);
+    } else {
+    }
+  }
+
+  dPrint("%s rowNum:%d %s", GREEN, rowNum, NC);
+  dPrint("%s threadNum:%d %s", GREEN, threadNum, NC);
+  dPrint("%s replica:%d %s", GREEN, replica, NC);
+}
+
+int main(int argc, char *argv[]) {
+  shellParseArgument(argc, argv);
+
+  taos_init();
+
+  ThreadObj *threads = calloc(threadNum, sizeof(ThreadObj));
+  for (int i = 0; i < threadNum; ++i) {
+    ThreadObj *    pthread = threads + i;
+    pthread_attr_t thattr;
+    pthread->index = i;
+    pthread_attr_init(&thattr);
+    pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+    pthread_create(&pthread->pid, &thattr, taos_execute, pthread);
+  }
+
+  for (int i = 0; i < threadNum; i++) {
+    pthread_join(threads[i].pid, NULL);
+  }
+
+  printf("all finished\n");
+
+  return 0;
+}
+
+void taos_error(TAOS *con) {
+  fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
+  taos_close(con);
+  exit(1);
+}
+
+void* taos_execute(void *param) {
+  ThreadObj *pThread = (ThreadObj *)param;
+
+  void *taos = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
+  if (taos == NULL) taos_error(taos);
+
+  char sql[1024] = {0};
+  sprintf(sql, "create database if not exists db replica %d", replica);
+  taos_query(taos, sql);
+
+  sprintf(sql, "create table if not exists db.t%d (ts timestamp, i int, j float, k double)", pThread->index);
+  taos_query(taos, sql);
+
+  int64_t timestamp = 1530374400000L;
+
+  sprintf(sql, "insert into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, 0, 0, 0);
+  int code = taos_query(taos, sql);
+  if (code != 0) printf("error code:%d, sql:%s\n", code, sql);
+  int affectrows = taos_affected_rows(taos);
+  if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql);
+
+  timestamp -= 1000;
+
+  int total_affect_rows = affectrows;
+
+  for (int i = 1; i < rowNum; ++i) {
+    sprintf(sql, "import into db.t%d values(%ld, %d, %d, %d)", pThread->index, timestamp, i, i, i);
+    code = taos_query(taos, sql);
+    if (code != 0) printf("error code:%d, sql:%s\n", code, sql);
+    int affectrows = taos_affected_rows(taos);
+    if (affectrows != 1) printf("affect rows:%d, sql:%s\n", affectrows, sql);
+
+    total_affect_rows += affectrows;
+
+    timestamp -= 1000;
+  }
+
+  printf("thread:%d run finished total_affect_rows:%d\n", pThread->index, total_affect_rows);
+
+  return NULL;
+}