diff --git a/include/os/os.h b/include/os/os.h index 7b478c3137612ecb87540d22dd4338e4118bf581..d96f7efb51dd0ca36d307308ba005d4fd7f628b6 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -59,6 +59,7 @@ extern "C" { #include "osEndian.h" #include "osEnv.h" #include "osFile.h" +#include "osLocale.h" #include "osLz4.h" #include "osMath.h" #include "osMemory.h" diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 63ccdb1a6404981efb593185e3f5497f9f34d132..c7c8ba9053427a6e4eca8e6713c59f21ac29a4e3 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -48,6 +48,8 @@ char *osDataDir(); char *osName(); char *osTimezone(); int8_t osDaylight(); +char *osLocale(); +char *osCharset(); void osSetTimezone(const char*timezone); diff --git a/include/common/tlocale.h b/include/os/osLocale.h similarity index 70% rename from include/common/tlocale.h rename to include/os/osLocale.h index d809c75d67983fb6ac844a656a361876ea7d3504..6e313eb8cde14e67f56a9e8334239ca714b88afd 100644 --- a/include/common/tlocale.h +++ b/include/os/osLocale.h @@ -13,17 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_LOCALE_H_ -#define _TD_COMMON_LOCALE_H_ +#ifndef _TD_OS_LOCALE_H_ +#define _TD_OS_LOCALE_H_ + +#include "os.h" +#include "osString.h" #ifdef __cplusplus extern "C" { #endif -void tsSetLocale(); +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +void taosSetSystemLocale(const char *inLocale, const char *inCharSet); #ifdef __cplusplus } -#endif /*_TD_COMMON_LOCALE_H_*/ - #endif + +#endif /*_TD_OS_LOCALE_H_*/ diff --git a/include/os/osString.h b/include/os/osString.h index 582411d4440e7b1539c4b926413c7ed9751dfe62..88160dd69e419450d1ec481a83a28065df99186d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 3780131cda913cd39f4b33f958962fee8d725f4c..64362603fea44648d159b1b2e3491c245066672f 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -42,8 +42,6 @@ extern int64_t tsOpenMax; extern int64_t tsStreamMax; extern int32_t tsNumOfCores; extern int32_t tsTotalMemoryMB; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index 6bf52a41085e1e9ba1626ff50eaaa1a19aad6ab1..ff015ef0b1221eeba361818f5e010bd7f43fb0fd 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,11 +20,11 @@ extern "C" { #endif -void osGetSystemTimezone(char *outTimezone); -void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); +void taosGetSystemTimezone(char *outTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); #ifdef __cplusplus } #endif -#endif /*_TD_OS_SOCKET_H_*/ +#endif /*_TD_OS_TIMEZONE_H_*/ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e26dbb639a66ebc530e0a41ae9afa702fb598420..b4c9efd77d6a93ea755cfa0902e54eb6a38561e9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -21,7 +21,6 @@ #include "tcompare.h" #include "tep.h" #include "tglobal.h" -#include "tlocale.h" #include "tlog.h" #include "tutil.h" #include "ulog.h" diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c deleted file mode 100644 index 1f2b9cd4b15cd7b633847d392d5cf570b834279a..0000000000000000000000000000000000000000 --- a/source/common/src/tlocale.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tutil.h" - -/** - * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. - * - * In case that the setLocale failed to be executed, the right charset needs to be set. - */ -void tsSetLocale() { - char *locale = setlocale(LC_CTYPE, tsLocale); - - // default locale or user specified locale is not valid, abort launch - if (locale == NULL) { - uError("Invalid locale:%s, please set the valid locale in config file", tsLocale); - } - - if (strlen(tsCharset) == 0) { - uError("failed to get charset, please set the valid charset in config file"); - exit(-1); - } - - if (!taosValidateEncodec(tsCharset)) { - uError("Invalid charset:%s, please set the valid charset in config file", tsCharset); - exit(-1); - } -} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 5da3233b1705d921bbe54e2f76f62e5123b563fc..05edf83638c74376c4685f304b995fbd7e2ae6c9 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" -#include "tlocale.h" static int32_t dmnAddEpCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; @@ -105,7 +104,6 @@ static int32_t dmnCheckCfg(SConfig *pCfg) { taosGetSystemInfo(); - tsSetLocale(); if (tsNumOfCores <= 0) { tsNumOfCores = 1; diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c index f877b22046e4e5cf0620513e389881a217efecab..b23826ceacc696669af94c2dbc729897e2fa5efd 100644 --- a/source/libs/config/src/config.c +++ b/source/libs/config/src/config.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "cfgInt.h" #include "tep.h" -#include "tlocale.h" #include "tmsg.h" #define CFG_NAME_PRINT_LEN 22 diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 462f7044e0d25b8c60c3c7efdee3c953486a6edd..8dd65fde14075811e993bf18cada8c6027fa245b 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -22,7 +22,8 @@ char configDir[PATH_MAX] = {0}; SOsEnv *osEnv() { return &env; } void osInitImp() { - osGetSystemTimezone(env.timezone); + taosGetSystemLocale(env.locale, env.charset); + taosGetSystemTimezone(env.timezone); osSetTimezone(env.timezone); } @@ -45,10 +46,12 @@ char *osTempDir() { return env.tempDir; } char *osDataDir() { return env.dataDir; } char *osName() { return env.osName; } char *osTimezone() { return env.timezone; } +char *osLocale() { return env.locale; } +char *osCharset() { return env.charset; } int8_t osDaylight() { return env.daylight; } -void osSetTimezone(const char *timezone) { osSetSystemTimezone(timezone, env.timezone, &env.daylight); } +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, env.timezone, &env.daylight); } #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c new file mode 100644 index 0000000000000000000000000000000000000000..e006a337a0817a9f94d0d694ead331c20d4a1dff --- /dev/null +++ b/source/os/src/osLocale.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osLocale.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + {"utf8", "UTF-8"}, + {"936", "CP936"}, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +/** + * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of + * both the TDengine Server and the Client may be interrupted. + * + * In case that the setLocale failed to be executed, the right charset needs to be set. + */ +void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { + char *locale = setlocale(LC_CTYPE, inLocale); + + // default locale or user specified locale is not valid, abort launch + if (inLocale == NULL) { + printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale); + } + + if (strlen(inCharSet) == 0) { + printf("failed to get charset, please set the valid charset in config file\n"); + exit(-1); + } + + if (!taosValidateEncodec(inCharSet)) { + printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + exit(-1); + } +} + +void taosGetSystemLocale(char *outLocale, char *outCharset) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + } + strcpy(outCharset, "cp936"); + +#elif defined(_TD_DARWIN_64) + /* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#else + /* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + // if user does not specify the charset, extract it from locale + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#endif +} diff --git a/source/os/src/osString.c b/source/os/src/osString.c index f714fd0dc4d26cf9f5c1391b34c6277b971cd74d..88ea4b3e1521103b4827c7b5646ec2ed92b556e6 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -18,24 +18,6 @@ #include #include -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} int64_t taosStr2int64(const char *str) { char *endptr = NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 439a8146abecc35b241821bc94d8e18d792152b3..4a8cc5ecbc92656bd372af9684039339ccca33c8 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -21,9 +21,6 @@ int64_t tsPageSize = 0; int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; int32_t tsNumOfCores = 1; -char tsLocale[TD_LOCALE_LEN] = {0}; -char tsCharset[TD_LOCALE_LEN] = {0}; // default encode string - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -89,17 +86,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { } - -static void taosGetSystemLocale() { - // get and set default locale - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TD_LOCALE_LEN); - } - - strcpy(tsCharset, "cp936"); -} - int32_t taosGetCpuCores() { SYSTEM_INFO info; GetSystemInfo(&info); @@ -191,7 +177,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemLocale(); } void taosKillSystem() { @@ -263,52 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TD_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} void taosKillSystem() { //printf("function taosKillSystem, exit!"); @@ -325,8 +264,6 @@ void taosGetSystemInfo() { long page_size = sysconf(_SC_PAGESIZE); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsPageSize = page_size; - - taosGetSystemLocale(); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -563,51 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { return true; } -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TD_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - // if user does not specify the charset, extract it from locale - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } @@ -836,7 +728,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemLocale(); } void taosKillSystem() { diff --git a/source/os/src/osTimeZone1.c b/source/os/src/osTimezone.c similarity index 97% rename from source/os/src/osTimeZone1.c rename to source/os/src/osTimezone.c index 84e86662d3355ee6b0cefd6ca801642c91a47696..a0ea01596fe5c811582073ec0809f66855d54f6e 100644 --- a/source/os/src/osTimeZone1.c +++ b/source/os/src/osTimezone.c @@ -45,7 +45,7 @@ #include #endif -void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { #ifdef WINDOWS char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", inTimezone); @@ -84,7 +84,7 @@ void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outD *outDaylight = daylight; } -void osGetSystemTimezone(char *outTimezone) { +void taosGetSystemTimezone(char *outTimezone) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) {