diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 1e59396c702dfa9c67eee09a3aed5cc6b421abf4..a4db6fd5fbc3e867cd30d2a1c871e48a7bfbd5eb 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -9,6 +9,7 @@ ADD_SUBDIRECTORY(lz4) ADD_SUBDIRECTORY(cJson) ADD_SUBDIRECTORY(wepoll) ADD_SUBDIRECTORY(MsvcLibX) +ADD_SUBDIRECTORY(rmonotonic) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) diff --git a/deps/MsvcLibX/include/msvcTime.h b/deps/MsvcLibX/include/msvcTime.h index 1897da58570cf503b05502d0a15c899f1f126015..6f8b5dac8f88e4c3dd40445f1c5512ba2e6e796f 100644 --- a/deps/MsvcLibX/include/msvcTime.h +++ b/deps/MsvcLibX/include/msvcTime.h @@ -38,6 +38,7 @@ typedef int clockid_t; /* Supported values for clockid_t */ #define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 int clock_gettime(clockid_t clock_id, struct timespec *tp); diff --git a/deps/MsvcLibX/src/clock_gettime.c b/deps/MsvcLibX/src/clock_gettime.c index fee7532e2e552ec08b615e61e3f4d79276ea6cfc..cbddf7107ae3733186c014c26a769f3f9d386d0e 100644 --- a/deps/MsvcLibX/src/clock_gettime.c +++ b/deps/MsvcLibX/src/clock_gettime.c @@ -34,15 +34,56 @@ #include "msvcTime.h" #include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */ -int clock_gettime(clockid_t clock_id, struct timespec *pTS) { - FILETIME ft; - if (clock_id != CLOCK_REALTIME) { - errno = EINVAL; - return -1; +#define MS_PER_SEC 1000ULL // MS = milliseconds +#define US_PER_MS 1000ULL // US = microseconds +#define HNS_PER_US 10ULL // HNS = hundred-nanoseconds (e.g., 1 hns = 100 ns) +#define NS_PER_US 1000ULL + +#define HNS_PER_SEC (MS_PER_SEC * US_PER_MS * HNS_PER_US) +#define NS_PER_HNS (100ULL) // NS = nanoseconds +#define NS_PER_SEC (MS_PER_SEC * US_PER_MS * NS_PER_US) + +int clock_gettime_monotonic(struct timespec *tv) { + static LARGE_INTEGER ticksPerSec; + LARGE_INTEGER ticks; + double seconds; + + if (!ticksPerSec.QuadPart) { + QueryPerformanceFrequency(&ticksPerSec); + if (!ticksPerSec.QuadPart) { + errno = ENOTSUP; + return -1; + } } + + QueryPerformanceCounter(&ticks); + + seconds = (double) ticks.QuadPart / (double) ticksPerSec.QuadPart; + tv->tv_sec = (time_t)seconds; + tv->tv_nsec = (long)((ULONGLONG)(seconds * NS_PER_SEC) % NS_PER_SEC); + + return 0; +} + +int clock_gettime_realtime(struct timespec *pTS) { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); Filetime2Timespec(&ft, pTS); + return 0; } +int clock_gettime(clockid_t clock_id, struct timespec *pTS) { + if (clock_id == CLOCK_MONOTONIC) { + return clock_gettime_monotonic(pTS); + } else if (clock_id == CLOCK_REALTIME) { + return clock_gettime_realtime(pTS); + } + + errno = ENOTSUP; + + return -1; +} + #endif /* defined(_WIN32) */ diff --git a/deps/rmonotonic/CMakeLists.txt b/deps/rmonotonic/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b870b7d6d7a3817c38bfd4c488f75890fd115b7f --- /dev/null +++ b/deps/rmonotonic/CMakeLists.txt @@ -0,0 +1,11 @@ +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) + +add_definitions(-DUSE_PROCESSOR_CLOCK) + +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../MsvcLibX/include) + +ADD_LIBRARY(rmonotonic ${SOURCE_LIST}) +TARGET_INCLUDE_DIRECTORIES(rmonotonic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) +IF (TD_WINDOWS) + TARGET_LINK_LIBRARIES(rmonotonic MsvcLibXw) +ENDIF () diff --git a/deps/rmonotonic/inc/monotonic.h b/deps/rmonotonic/inc/monotonic.h new file mode 100644 index 0000000000000000000000000000000000000000..0850548d36629b0b1bfba04e818937e275cac5bf --- /dev/null +++ b/deps/rmonotonic/inc/monotonic.h @@ -0,0 +1,56 @@ +#ifndef __MONOTONIC_H +#define __MONOTONIC_H +/* The monotonic clock is an always increasing clock source. It is unrelated to + * the actual time of day and should only be used for relative timings. The + * monotonic clock is also not guaranteed to be chronologically precise; there + * may be slight skew/shift from a precise clock. + * + * Depending on system architecture, the monotonic time may be able to be + * retrieved much faster than a normal clock source by using an instruction + * counter on the CPU. On x86 architectures (for example), the RDTSC + * instruction is a very fast clock source for this purpose. + */ + +//#include "fmacros.h" +#include +//#include + +#if defined(_WIN32) || defined(_WIN64) + #define inline +#endif + +/* A counter in micro-seconds. The 'monotime' type is provided for variables + * holding a monotonic time. This will help distinguish & document that the + * variable is associated with the monotonic clock and should not be confused + * with other types of time.*/ +typedef uint64_t monotime; + +/* Retrieve counter of micro-seconds relative to an arbitrary point in time. */ +extern monotime (*getMonotonicUs)(void); + + +/* Call once at startup to initialize the monotonic clock. Though this only + * needs to be called once, it may be called additional times without impact. + * Returns a printable string indicating the type of clock initialized. + * (The returned string is static and doesn't need to be freed.) */ +const char * monotonicInit(); + + +/* Functions to measure elapsed time. Example: + * monotime myTimer; + * elapsedStart(&myTimer); + * while (elapsedMs(myTimer) < 10) {} // loops for 10ms + */ +static inline void elapsedStart(monotime *start_time) { + *start_time = getMonotonicUs(); +} + +static inline uint64_t elapsedUs(monotime start_time) { + return getMonotonicUs() - start_time; +} + +static inline uint64_t elapsedMs(monotime start_time) { + return elapsedUs(start_time) / 1000; +} + +#endif diff --git a/deps/rmonotonic/src/monotonic.c b/deps/rmonotonic/src/monotonic.c new file mode 100644 index 0000000000000000000000000000000000000000..622ac1b075a66ec49338cda3cc4afbc8355636dd --- /dev/null +++ b/deps/rmonotonic/src/monotonic.c @@ -0,0 +1,173 @@ +#include "monotonic.h" +#include +#include +#include +#include + +#undef NDEBUG +#include + +#if defined(_WIN32) || defined(_WIN64) +#include "msvcTime.h" +#include "msvcStdio.h" +#endif + +/* The function pointer for clock retrieval. */ +monotime (*getMonotonicUs)(void) = NULL; + +static char monotonic_info_string[32]; + + +/* Using the processor clock (aka TSC on x86) can provide improved performance + * throughout Redis wherever the monotonic clock is used. The processor clock + * is significantly faster than calling 'clock_getting' (POSIX). While this is + * generally safe on modern systems, this link provides additional information + * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage + * + * To use the processor clock, either uncomment this line, or build with + * CFLAGS="-DUSE_PROCESSOR_CLOCK" +#define USE_PROCESSOR_CLOCK + */ + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) +#include +#include + +static long mono_ticksPerMicrosecond = 0; + +static monotime getMonotonicUs_x86() { + return __rdtsc() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_x86linux() { + const int bufflen = 256; + char buf[bufflen]; + regex_t cpuGhzRegex, constTscRegex; + const size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + int constantTsc = 0; + int rc; + + /* Determine the number of TSC ticks in a micro-second. This is + * a constant value matching the standard speed of the processor. + * On modern processors, this speed remains constant even though + * the actual clock speed varies dynamically for each core. */ + rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED); + assert(rc == 0); + + /* Also check that the constant_tsc flag is present. (It should be + * unless this is a really old CPU. */ + rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED); + assert(rc == 0); + + FILE *cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) { + buf[pmatch[1].rm_eo] = '\0'; + double ghz = atof(&buf[pmatch[1].rm_so]); + mono_ticksPerMicrosecond = (long)(ghz * 1000); + break; + } + } + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) { + constantTsc = 1; + break; + } + } + + fclose(cpuinfo); + } + regfree(&cpuGhzRegex); + regfree(&constTscRegex); + + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate"); + return; + } + if (!constantTsc) { + fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_x86; +} +#endif + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) +static long mono_ticksPerMicrosecond = 0; + +/* Read the clock value. */ +static inline uint64_t __cntvct() { + uint64_t virtual_timer_value; + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +/* Read the Count-timer Frequency. */ +static inline uint32_t cntfrq_hz() { + uint64_t virtual_freq_value; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value)); + return (uint32_t)virtual_freq_value; /* top 32 bits are reserved */ +} + +static monotime getMonotonicUs_aarch64() { + return __cntvct() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_aarch64() { + mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L; + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: aarch64, unable to determine clock rate"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_aarch64; +} +#endif + +static monotime getMonotonicUs_posix(void) { + /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems + * did not support this until much later. CLOCK_MONOTONIC is technically + * optional and may not be supported - but it appears to be universal. + * If this is not supported, provide a system-specific alternate version. */ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; +} + +static void monotonicInit_posix() { + /* Ensure that CLOCK_MONOTONIC is supported. This should be supported + * on any reasonably current OS. If the assertion below fails, provide + * an appropriate alternate implementation. */ + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(rc == 0); + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "POSIX clock_gettime"); + getMonotonicUs = getMonotonicUs_posix; +} + + + +const char * monotonicInit() { + #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) + if (getMonotonicUs == NULL) monotonicInit_x86linux(); + #endif + + #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) + if (getMonotonicUs == NULL) monotonicInit_aarch64(); + #endif + + if (getMonotonicUs == NULL) monotonicInit_posix(); + + return monotonic_info_string; +} diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index e74a61801bd1836c468c2c3d03746b58c50083e7..9cec9963af6f8abdcaedeeb4e7d40f9244508be2 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -168,6 +168,7 @@ function install_main_path() { if [ "$verMode" == "cluster" ]; then ${csudo} mkdir -p ${nginx_dir} fi + ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: } function install_bin() { diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index 30115907b6d204d39d2217f289390dc3437cc938..394fd7debe6fe9929b2b2df0d20a19df3e405793 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -18,6 +18,9 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" +#include +#include + static void taosGetSystemTimezone() { // get and set default timezone @@ -103,8 +106,18 @@ int taosSystem(const char *cmd) { void taosSetCoreDump() {} +char cmdline[1024]; + char *taosGetCmdlineByPID(int pid) { - return "[not supported yet]"; + + errno = 0; + + if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { + fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); + return strerror(errno); + } + + return cmdline; } bool taosGetSystemUid(char *uid) { diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 95b618507a0fd0ae3601f04c86f47a1a3eb98cc1..80e874ad92cebc267460c1e59e494fa52f004ced 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -3,9 +3,10 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) -TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) +TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 015c687b3d941db3f6414d0d3d92564dd14e0fdb..6029edf5120314cf13405f69fb8e120e73d61183 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -18,6 +18,7 @@ #include "tsched.h" #include "ttimer.h" #include "tutil.h" +#include "monotonic.h" extern int8_t tscEmbedded; @@ -186,6 +187,10 @@ static void removeTimer(uintptr_t id) { unlockTimerList(list); } +static int64_t getMonotonicMs(void) { + return (int64_t) getMonotonicUs() / 1000; +} + static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timerAddRef(timer); // select a wheel for the timer, we are not an accurate timer, @@ -201,7 +206,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { time_wheel_t* wheel = wheels + timer->wheel; timer->prev = NULL; - timer->expireAt = taosGetTimestampMs() + delay; + timer->expireAt = getMonotonicMs() + delay; pthread_mutex_lock(&wheel->mutex); @@ -334,7 +339,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle } static void taosTimerLoopFunc(int signo) { - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. @@ -501,7 +506,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { @@ -532,6 +537,8 @@ static void taosTmrModuleInit(void) { } void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { + tmrInfo("ttimer monotonic clock source:%s", monotonicInit()); + pthread_once(&tmrModuleInit, taosTmrModuleInit); pthread_mutex_lock(&tmrCtrlMutex); diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 3be784ae95c84599c773d5c52b68ef5a630482ea..aee8f7502c354628d9cb0ca9df55ab4a7ad17c6f 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -20,6 +20,7 @@ python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py #python3 ./test.py -f insert/before_1970.py +python3 ./test.py -f insert/metadataUpdate.py python3 bug2265.py #table diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py new file mode 100644 index 0000000000000000000000000000000000000000..c3ea5c6a7e7b5dbff9ded75394793e7fc4a7fd6c --- /dev/null +++ b/tests/pytest/insert/metadataUpdate.py @@ -0,0 +1,67 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from multiprocessing import Process + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def updateMetadata(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = tdDnodes.getSimCfgPath() + + self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) + self.cursor = self.conn.cursor() + self.cursor.execute("alter table db.tb add column col2 int") + print("alter table done") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table if not exists tb (ts timestamp, col1 int)") + tdSql.execute("insert into tb values(%d, 1)" % self.ts) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + p = Process(target=self.updateMetadata, args=()) + p.start() + p.join() + p.terminate() + + tdSql.execute("insert into tb values(%d, 1, 2)" % (self.ts + 1)) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase())