提交 77788309 编写于 作者: S Shengliang Guan

locale

上级 9aaa9d20
...@@ -59,6 +59,7 @@ extern "C" { ...@@ -59,6 +59,7 @@ extern "C" {
#include "osEndian.h" #include "osEndian.h"
#include "osEnv.h" #include "osEnv.h"
#include "osFile.h" #include "osFile.h"
#include "osLocale.h"
#include "osLz4.h" #include "osLz4.h"
#include "osMath.h" #include "osMath.h"
#include "osMemory.h" #include "osMemory.h"
......
...@@ -48,6 +48,8 @@ char *osDataDir(); ...@@ -48,6 +48,8 @@ char *osDataDir();
char *osName(); char *osName();
char *osTimezone(); char *osTimezone();
int8_t osDaylight(); int8_t osDaylight();
char *osLocale();
char *osCharset();
void osSetTimezone(const char*timezone); void osSetTimezone(const char*timezone);
......
...@@ -13,17 +13,22 @@ ...@@ -13,17 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_COMMON_LOCALE_H_ #ifndef _TD_OS_LOCALE_H_
#define _TD_COMMON_LOCALE_H_ #define _TD_OS_LOCALE_H_
#include "os.h"
#include "osString.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
void tsSetLocale(); char *taosCharsetReplace(char *charsetstr);
void taosGetSystemLocale(char *outLocale, char *outCharset);
void taosSetSystemLocale(const char *inLocale, const char *inCharSet);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif /*_TD_COMMON_LOCALE_H_*/
#endif #endif
#endif /*_TD_OS_LOCALE_H_*/
...@@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); ...@@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
bool taosValidateEncodec(const char *encodec); bool taosValidateEncodec(const char *encodec);
char * taosCharsetReplace(char *charsetstr);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -42,8 +42,6 @@ extern int64_t tsOpenMax; ...@@ -42,8 +42,6 @@ extern int64_t tsOpenMax;
extern int64_t tsStreamMax; extern int64_t tsStreamMax;
extern int32_t tsNumOfCores; extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB; extern int32_t tsTotalMemoryMB;
extern char tsLocale[];
extern char tsCharset[]; // default encode string
......
...@@ -20,11 +20,11 @@ ...@@ -20,11 +20,11 @@
extern "C" { extern "C" {
#endif #endif
void osGetSystemTimezone(char *outTimezone); void taosGetSystemTimezone(char *outTimezone);
void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_OS_SOCKET_H_*/ #endif /*_TD_OS_TIMEZONE_H_*/
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
#include "tcompare.h" #include "tcompare.h"
#include "tep.h" #include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tlocale.h"
#include "tlog.h" #include "tlog.h"
#include "tutil.h" #include "tutil.h"
#include "ulog.h" #include "ulog.h"
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tglobal.h"
#include "tutil.h"
/**
* In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
* both the TDengine Server and the Client may be interrupted.
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
void tsSetLocale() {
char *locale = setlocale(LC_CTYPE, tsLocale);
// default locale or user specified locale is not valid, abort launch
if (locale == NULL) {
uError("Invalid locale:%s, please set the valid locale in config file", tsLocale);
}
if (strlen(tsCharset) == 0) {
uError("failed to get charset, please set the valid charset in config file");
exit(-1);
}
if (!taosValidateEncodec(tsCharset)) {
uError("Invalid charset:%s, please set the valid charset in config file", tsCharset);
exit(-1);
}
}
\ No newline at end of file
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmnInt.h" #include "dmnInt.h"
#include "tlocale.h"
static int32_t dmnAddEpCfg(SConfig *pCfg) { static int32_t dmnAddEpCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0}; char defaultFqdn[TSDB_FQDN_LEN] = {0};
...@@ -105,7 +104,6 @@ static int32_t dmnCheckCfg(SConfig *pCfg) { ...@@ -105,7 +104,6 @@ static int32_t dmnCheckCfg(SConfig *pCfg) {
taosGetSystemInfo(); taosGetSystemInfo();
tsSetLocale();
if (tsNumOfCores <= 0) { if (tsNumOfCores <= 0) {
tsNumOfCores = 1; tsNumOfCores = 1;
......
...@@ -16,7 +16,6 @@ ...@@ -16,7 +16,6 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "cfgInt.h" #include "cfgInt.h"
#include "tep.h" #include "tep.h"
#include "tlocale.h"
#include "tmsg.h" #include "tmsg.h"
#define CFG_NAME_PRINT_LEN 22 #define CFG_NAME_PRINT_LEN 22
......
...@@ -22,7 +22,8 @@ char configDir[PATH_MAX] = {0}; ...@@ -22,7 +22,8 @@ char configDir[PATH_MAX] = {0};
SOsEnv *osEnv() { return &env; } SOsEnv *osEnv() { return &env; }
void osInitImp() { void osInitImp() {
osGetSystemTimezone(env.timezone); taosGetSystemLocale(env.locale, env.charset);
taosGetSystemTimezone(env.timezone);
osSetTimezone(env.timezone); osSetTimezone(env.timezone);
} }
...@@ -45,10 +46,12 @@ char *osTempDir() { return env.tempDir; } ...@@ -45,10 +46,12 @@ char *osTempDir() { return env.tempDir; }
char *osDataDir() { return env.dataDir; } char *osDataDir() { return env.dataDir; }
char *osName() { return env.osName; } char *osName() { return env.osName; }
char *osTimezone() { return env.timezone; } char *osTimezone() { return env.timezone; }
char *osLocale() { return env.locale; }
char *osCharset() { return env.charset; }
int8_t osDaylight() { return env.daylight; } int8_t osDaylight() { return env.daylight; }
void osSetTimezone(const char *timezone) { osSetSystemTimezone(timezone, env.timezone, &env.daylight); } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, env.timezone, &env.daylight); }
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "osLocale.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#if (_WIN64)
#include <iphlpapi.h>
#include <mswsock.h>
#include <psapi.h>
#include <stdio.h>
#include <windows.h>
#include <ws2tcpip.h>
#pragma comment(lib, "Mswsock.lib ")
#endif
#include <objbase.h>
#pragma warning(push)
#pragma warning(disable : 4091)
#include <DbgHelp.h>
#pragma warning(pop)
#elif defined(_TD_DARWIN_64)
#include <errno.h>
#include <libproc.h>
#else
#include <argp.h>
#include <linux/sysctl.h>
#include <sys/file.h>
#include <sys/resource.h>
#include <sys/statvfs.h>
#include <sys/syscall.h>
#include <sys/utsname.h>
#include <unistd.h>
#endif
typedef struct CharsetPair {
char *oldCharset;
char *newCharset;
} CharsetPair;
char *taosCharsetReplace(char *charsetstr) {
CharsetPair charsetRep[] = {
{"utf8", "UTF-8"},
{"936", "CP936"},
};
for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
return strdup(charsetRep[i].newCharset);
}
}
return strdup(charsetstr);
}
/**
* In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
* both the TDengine Server and the Client may be interrupted.
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
char *locale = setlocale(LC_CTYPE, inLocale);
// default locale or user specified locale is not valid, abort launch
if (inLocale == NULL) {
printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale);
}
if (strlen(inCharSet) == 0) {
printf("failed to get charset, please set the valid charset in config file\n");
exit(-1);
}
if (!taosValidateEncodec(inCharSet)) {
printf("Invalid charset:%s, please set the valid charset in config file", inCharSet);
exit(-1);
}
}
void taosGetSystemLocale(char *outLocale, char *outCharset) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
}
strcpy(outCharset, "cp936");
#elif defined(_TD_DARWIN_64)
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN);
free(revisedCharset);
// printf("charset not configured, set to system default:%s", outCharset);
} else {
strcpy(outCharset, "UTF-8");
// printf("can't get locale and charset from system, set it to UTF-8");
}
#else
/*
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
// if user does not specify the charset, extract it from locale
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
// printf("charset not configured, set to system default:%s", outCharset);
} else {
strcpy(outCharset, "UTF-8");
// printf("can't get locale and charset from system, set it to UTF-8");
}
#endif
}
...@@ -18,24 +18,6 @@ ...@@ -18,24 +18,6 @@
#include <wchar.h> #include <wchar.h>
#include <wctype.h> #include <wctype.h>
typedef struct CharsetPair {
char *oldCharset;
char *newCharset;
} CharsetPair;
char *taosCharsetReplace(char *charsetstr) {
CharsetPair charsetRep[] = {
{ "utf8", "UTF-8" }, { "936", "CP936" },
};
for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
return strdup(charsetRep[i].newCharset);
}
}
return strdup(charsetstr);
}
int64_t taosStr2int64(const char *str) { int64_t taosStr2int64(const char *str) {
char *endptr = NULL; char *endptr = NULL;
......
...@@ -21,9 +21,6 @@ int64_t tsPageSize = 0; ...@@ -21,9 +21,6 @@ int64_t tsPageSize = 0;
int64_t tsOpenMax = 0; int64_t tsOpenMax = 0;
int64_t tsStreamMax = 0; int64_t tsStreamMax = 0;
int32_t tsNumOfCores = 1; int32_t tsNumOfCores = 1;
char tsLocale[TD_LOCALE_LEN] = {0};
char tsCharset[TD_LOCALE_LEN] = {0}; // default encode string
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
/* /*
...@@ -89,17 +86,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { ...@@ -89,17 +86,6 @@ bool taosGetProcMemory(float *memoryUsedMB) {
} }
static void taosGetSystemLocale() {
// get and set default locale
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
}
strcpy(tsCharset, "cp936");
}
int32_t taosGetCpuCores() { int32_t taosGetCpuCores() {
SYSTEM_INFO info; SYSTEM_INFO info;
GetSystemInfo(&info); GetSystemInfo(&info);
...@@ -191,7 +177,6 @@ void taosGetSystemInfo() { ...@@ -191,7 +177,6 @@ void taosGetSystemInfo() {
taosGetCpuUsage(&tmp1, &tmp2); taosGetCpuUsage(&tmp1, &tmp2);
taosGetProcIO(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2);
taosGetSystemLocale();
} }
void taosKillSystem() { void taosKillSystem() {
...@@ -263,52 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } ...@@ -263,52 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; }
#include <errno.h> #include <errno.h>
#include <libproc.h> #include <libproc.h>
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
static void taosGetSystemLocale() { // get and set default locale
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
//printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
//printf("locale not configured, set to system default:%s", tsLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(tsLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
//printf("charset not configured, set to system default:%s", tsCharset);
} else {
strcpy(tsCharset, "UTF-8");
//printf("can't get locale and charset from system, set it to UTF-8");
}
}
void taosKillSystem() { void taosKillSystem() {
//printf("function taosKillSystem, exit!"); //printf("function taosKillSystem, exit!");
...@@ -325,8 +264,6 @@ void taosGetSystemInfo() { ...@@ -325,8 +264,6 @@ void taosGetSystemInfo() {
long page_size = sysconf(_SC_PAGESIZE); long page_size = sysconf(_SC_PAGESIZE);
tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024);
tsPageSize = page_size; tsPageSize = page_size;
taosGetSystemLocale();
} }
bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
...@@ -563,51 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { ...@@ -563,51 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
return true; return true;
} }
/*
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
static void taosGetSystemLocale() { // get and set default locale
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
//printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
//printf("locale not configured, set to system default:%s", tsLocale);
}
// if user does not specify the charset, extract it from locale
char *str = strrchr(tsLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
//printf("charset not configured, set to system default:%s", tsCharset);
} else {
strcpy(tsCharset, "UTF-8");
//printf("can't get locale and charset from system, set it to UTF-8");
}
}
int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
...@@ -836,7 +728,6 @@ void taosGetSystemInfo() { ...@@ -836,7 +728,6 @@ void taosGetSystemInfo() {
taosGetCpuUsage(&tmp1, &tmp2); taosGetCpuUsage(&tmp1, &tmp2);
taosGetProcIO(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2);
taosGetSystemLocale();
} }
void taosKillSystem() { void taosKillSystem() {
......
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
#include <unistd.h> #include <unistd.h>
#endif #endif
void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
#ifdef WINDOWS #ifdef WINDOWS
char winStr[TD_LOCALE_LEN * 2]; char winStr[TD_LOCALE_LEN * 2];
sprintf(winStr, "TZ=%s", inTimezone); sprintf(winStr, "TZ=%s", inTimezone);
...@@ -84,7 +84,7 @@ void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outD ...@@ -84,7 +84,7 @@ void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outD
*outDaylight = daylight; *outDaylight = daylight;
} }
void osGetSystemTimezone(char *outTimezone) { void taosGetSystemTimezone(char *outTimezone) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char *tz = getenv("TZ"); char *tz = getenv("TZ");
if (tz == NULL || strlen(tz) == 0) { if (tz == NULL || strlen(tz) == 0) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册