Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0

ci/scripts/check_ccache.sh

@@ -41,12 +41,12 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
     exit 1
 fi
 
-for BRANCH_NAME in ${BRANCH_NAMES}
-do
-    echo "fetching ${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
-    wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
+check_ccache() {
+    BRANCH=$1
+    echo "fetching ${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
+    wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
     if [[ $? == 0 ]];then
-        wget "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \
+        wget "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \
         mkdir -p ${CCACHE_DIRECTORY} && \
         tar zxf ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz -C ${CCACHE_DIRECTORY} && \
         rm ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz
@@ -55,21 +55,18 @@ do
             exit 0
         fi
     fi
+}
+
+for BRANCH_NAME in ${BRANCH_NAMES}
+do
+    if [[ "${BRANCH_NAME}" != "HEAD" ]];then
+        check_ccache ${BRANCH_NAME}
+    fi
 done
 
 if [[ -n "${CHANGE_BRANCH}" && "${BRANCH_NAME}" =~ "PR-" ]];then
-    echo "fetching ${CHANGE_BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
-    wget -q --method HEAD "${ARTIFACTORY_URL}/${CHANGE_BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz"
-    if [[ $? == 0 ]];then
-        wget "${ARTIFACTORY_URL}/${CHANGE_BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \
-        mkdir -p ${CCACHE_DIRECTORY} && \
-        tar zxf ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz -C ${CCACHE_DIRECTORY} && \
-        rm ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz
-        if [[ $? == 0 ]];then
-            echo "found cache"
-            exit 0
-        fi
-    fi
+    check_ccache ${CHANGE_BRANCH}
+    check_ccache ${BRANCH_NAME}
 fi
 
 echo "could not download cache" && exit 1

ci/scripts/update_ccache.sh

@@ -56,14 +56,16 @@ REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"
 
 ccache --show-stats
 
-echo "Updating ccache package file: ${PACKAGE_FILE}"
-tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache .
-echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
-curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH}/${PACKAGE_FILE}
-if [[ $? == 0 ]];then
+if [[ "${BRANCH_NAME}" != "HEAD" ]];then
+    echo "Updating ccache package file: ${PACKAGE_FILE}"
+    tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache .
+    echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
+    curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH}/${PACKAGE_FILE}
+    if [[ $? == 0 ]];then
         echo "Uploading ccache package file success !"
         exit 0
-else
+    else
         echo "Uploading ccache package file fault !"
         exit 1
+    fi
 fi

core/CMakeLists.txt

@@ -187,7 +187,7 @@ endif ()
 add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
 
 if ("${MILVUS_DB_PATH}" STREQUAL "")
-    set(MILVUS_DB_PATH "/tmp/milvus")
+    set(MILVUS_DB_PATH "${CMAKE_INSTALL_PREFIX}")
 endif ()
 
 if (MILVUS_GPU_VERSION)

shards/utils/colors.py

+# Reset
+Color_Off='\033[0m'       # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+White='\033[0;37m'        # White
+
+# Bold
+BBlack='\033[1;30m'       # Black
+BRed='\033[1;31m'         # Red
+BGreen='\033[1;32m'       # Green
+BYellow='\033[1;33m'      # Yellow
+BBlue='\033[1;34m'        # Blue
+BPurple='\033[1;35m'      # Purple
+BCyan='\033[1;36m'        # Cyan
+BWhite='\033[1;37m'       # White
+
+# Underline
+UBlack='\033[4;30m'       # Black
+URed='\033[4;31m'         # Red
+UGreen='\033[4;32m'       # Green
+UYellow='\033[4;33m'      # Yellow
+UBlue='\033[4;34m'        # Blue
+UPurple='\033[4;35m'      # Purple
+UCyan='\033[4;36m'        # Cyan
+UWhite='\033[4;37m'       # White
+
+# Background
+On_Black='\033[40m'       # Black
+On_Red='\033[41m'         # Red
+On_Green='\033[42m'       # Green
+On_Yellow='\033[43m'      # Yellow
+On_Blue='\033[44m'        # Blue
+On_Purple='\033[45m'      # Purple
+On_Cyan='\033[46m'        # Cyan
+On_White='\033[47m'       # White
+
+# High Intensity
+IBlack='\033[0;90m'       # Black
+IRed='\033[0;91m'         # Red
+IGreen='\033[0;92m'       # Green
+IYellow='\033[0;93m'      # Yellow
+IBlue='\033[0;94m'        # Blue
+IPurple='\033[0;95m'      # Purple
+ICyan='\033[0;96m'        # Cyan
+IWhite='\033[0;97m'       # White
+
+# Bold High Intensity
+BIBlack='\033[1;90m'      # Black
+BIRed='\033[1;91m'        # Red
+BIGreen='\033[1;92m'      # Green
+BIYellow='\033[1;93m'     # Yellow
+BIBlue='\033[1;94m'       # Blue
+BIPurple='\033[1;95m'     # Purple
+BICyan='\033[1;96m'       # Cyan
+BIWhite='\033[1;97m'      # White
+
+# High Intensity backgrounds
+On_IBlack='\033[0;100m'   # Black
+On_IRed='\033[0;101m'     # Red
+On_IGreen='\033[0;102m'   # Green
+On_IYellow='\033[0;103m'  # Yellow
+On_IBlue='\033[0;104m'    # Blue
+On_IPurple='\033[0;105m'  # Purple
+On_ICyan='\033[0;106m'    # Cyan
+On_IWhite='\033[0;107m'   # White

shards/utils/logger_helper.py

@@ -3,6 +3,7 @@ import datetime
 from pytz import timezone
 from logging import Filter
 import logging.config
+from utils import colors
 
 
 class InfoFilter(logging.Filter):
@@ -31,29 +32,53 @@ class CriticalFilter(logging.Filter):
 
 
 COLORS = {
-    'HEADER': '\033[95m',
-    'INFO': '\033[92m',
-    'DEBUG': '\033[94m',
-    'WARNING': '\033[93m',
-    'ERROR': '\033[95m',
-    'CRITICAL': '\033[91m',
-    'ENDC': '\033[0m',
+    'HEADER': colors.BWhite,
+    'INFO': colors.On_IWhite + colors.BBlack,
+    'INFOM': colors.White,
+    'DEBUG': colors.On_IBlue + colors.BWhite,
+    'DEBUGM': colors.BIBlue,
+    'WARNING': colors.On_IYellow + colors.BWhite,
+    'WARNINGM': colors.BIYellow,
+    'ERROR': colors.On_IRed + colors.BWhite,
+    'ERRORM': colors.BIRed,
+    'CRITICAL': colors.On_Red + colors.BWhite,
+    'CRITICALM': colors.BRed,
+    'ASCTIME': colors.On_Cyan + colors.BIYellow,
+    'MESSAGE': colors.IGreen,
+    'FILENAME': colors.BCyan,
+    'LINENO': colors.BCyan,
+    'THREAD': colors.BCyan,
+    'ENDC': colors.Color_Off,
 }
 
 
 class ColorFulFormatColMixin:
     def format_col(self, message_str, level_name):
         if level_name in COLORS.keys():
-            message_str = COLORS.get(level_name) + message_str + COLORS.get(
-                'ENDC')
+            message_str = COLORS[level_name] + message_str + COLORS['ENDC']
         return message_str
 
+    def formatTime(self, record, datefmt=None):
+        ret =  super().formatTime(record, datefmt)
+        ret = COLORS['ASCTIME'] + ret + COLORS['ENDC']
+        return ret
 
-class ColorfulFormatter(logging.Formatter, ColorFulFormatColMixin):
+    def format_record(self, record):
+        msg_schema = record.levelname + 'M'
+        record.msg = '{}{}{}'.format(COLORS[msg_schema], record.msg, COLORS['ENDC'])
+        record.filename = COLORS['FILENAME'] + record.filename + COLORS['ENDC']
+        record.lineno = '{}{}{}'.format(COLORS['LINENO'], record.lineno, COLORS['ENDC'])
+        record.threadName = '{}{}{}'.format(COLORS['THREAD'], record.threadName, COLORS['ENDC'])
+        record.levelname = COLORS[record.levelname] + record.levelname + COLORS['ENDC']
+        return record
+
+
+class ColorfulFormatter(ColorFulFormatColMixin, logging.Formatter):
     def format(self, record):
+        record = self.format_record(record)
         message_str = super(ColorfulFormatter, self).format(record)
 
-        return self.format_col(message_str, level_name=record.levelname)
+        return message_str
 
 
 def config(log_level, log_path, name, tz='UTC'):
@@ -76,7 +101,9 @@ def config(log_level, log_path, name, tz='UTC'):
                 'format': '%(asctime)s | %(levelname)s | %(name)s | %(threadName)s: %(message)s (%(filename)s:%(lineno)s)',
             },
             'colorful_console': {
-                'format': '%(asctime)s | %(levelname)s | %(name)s | %(threadName)s: %(message)s (%(filename)s:%(lineno)s)',
+                'format': '%(asctime)s | %(levelname)s: %(message)s (%(filename)s:%(lineno)s) (%(threadName)s)',
+                # 'format': '%(asctime)s | %(levelname)s | %(threadName)s: %(message)s (%(filename)s:%(lineno)s)',
+                # 'format': '%(asctime)s | %(levelname)s | %(name)s | %(threadName)s: %(message)s (%(filename)s:%(lineno)s)',
                 '()': ColorfulFormatter,
             },
         },