locale

include/os/os.h

@@ -59,6 +59,7 @@ extern "C" {
 #include "osEndian.h"
 #include "osEnv.h"
 #include "osFile.h"
+#include "osLocale.h"
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"

include/os/osEnv.h

@@ -48,6 +48,8 @@ char   *osDataDir();
 char   *osName();
 char   *osTimezone();
 int8_t  osDaylight();
+char   *osLocale();
+char   *osCharset();
 
 void osSetTimezone(const char*timezone);
 

include/common/tlocale.h → include/os/osLocale.h

@@ -13,17 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_COMMON_LOCALE_H_
-#define _TD_COMMON_LOCALE_H_
+#ifndef _TD_OS_LOCALE_H_
+#define _TD_OS_LOCALE_H_
+
+#include "os.h"
+#include "osString.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void tsSetLocale();
+char *taosCharsetReplace(char *charsetstr);
+void  taosGetSystemLocale(char *outLocale, char *outCharset);
+void  taosSetSystemLocale(const char *inLocale, const char *inCharSet);
 
 #ifdef __cplusplus
 }
-#endif /*_TD_COMMON_LOCALE_H_*/
-
 #endif
+
+#endif /*_TD_OS_LOCALE_H_*/

include/os/osString.h

@@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
 bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
-char *  taosCharsetReplace(char *charsetstr);
 
 #ifdef __cplusplus
 }

include/os/osSysinfo.h

@@ -42,8 +42,6 @@ extern int64_t tsOpenMax;
 extern int64_t tsStreamMax;
 extern int32_t tsNumOfCores;
 extern int32_t tsTotalMemoryMB;
-extern char    tsLocale[];
-extern char    tsCharset[];  // default encode string
 
 
 

include/os/osTimezone.h

@@ -20,11 +20,11 @@
 extern "C" {
 #endif
 
-void osGetSystemTimezone(char *outTimezone);
-void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
+void taosGetSystemTimezone(char *outTimezone);
+void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_OS_SOCKET_H_*/
+#endif /*_TD_OS_TIMEZONE_H_*/

source/common/src/tglobal.c

@@ -21,7 +21,6 @@
 #include "tcompare.h"
 #include "tep.h"
 #include "tglobal.h"
-#include "tlocale.h"
 #include "tlog.h"
 #include "tutil.h"
 #include "ulog.h"

source/common/src/tlocale.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "ulog.h"
-#include "tglobal.h"
-#include "tutil.h"
-
-/**
- * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
- * both the TDengine Server and the Client may be interrupted.
- *
- * In case that the setLocale failed to be executed, the right charset needs to be set.
- */
-void tsSetLocale() {
-  char *locale = setlocale(LC_CTYPE, tsLocale);
-
-  // default locale or user specified locale is not valid, abort launch
-  if (locale == NULL) {
-    uError("Invalid locale:%s, please set the valid locale in config file", tsLocale);
-  }
-
-  if (strlen(tsCharset) == 0) {
-    uError("failed to get charset, please set the valid charset in config file");
-    exit(-1);
-  }
-
-  if (!taosValidateEncodec(tsCharset)) {
-    uError("Invalid charset:%s, please set the valid charset in config file", tsCharset);
-    exit(-1);
-  }
-}
\ No newline at end of file

source/dnode/mgmt/daemon/src/dmnCfg.c

@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "dmnInt.h"
-#include "tlocale.h"
 
 static int32_t dmnAddEpCfg(SConfig *pCfg) {
   char defaultFqdn[TSDB_FQDN_LEN] = {0};
@@ -105,7 +104,6 @@ static int32_t dmnCheckCfg(SConfig *pCfg) {
 
   taosGetSystemInfo();
 
-  tsSetLocale();
 
   if (tsNumOfCores <= 0) {
     tsNumOfCores = 1;

source/libs/config/src/config.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "cfgInt.h"
 #include "tep.h"
-#include "tlocale.h"
 #include "tmsg.h"
 
 #define CFG_NAME_PRINT_LEN 22

source/os/src/osEnv.c

@@ -22,7 +22,8 @@ char   configDir[PATH_MAX] = {0};
 SOsEnv *osEnv() { return &env; }
 
 void osInitImp() {
-  osGetSystemTimezone(env.timezone);
+  taosGetSystemLocale(env.locale, env.charset);
+  taosGetSystemTimezone(env.timezone);
   osSetTimezone(env.timezone);
 }
 
@@ -45,10 +46,12 @@ char *osTempDir() { return env.tempDir; }
 char *osDataDir() { return env.dataDir; }
 char *osName() { return env.osName; }
 char *osTimezone() { return env.timezone; }
+char *osLocale() { return env.locale; }
+char *osCharset() { return env.charset; }
 
 int8_t osDaylight() { return env.daylight; }
 
-void osSetTimezone(const char *timezone) { osSetSystemTimezone(timezone, env.timezone, &env.daylight); }
+void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, env.timezone, &env.daylight); }
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 

source/os/src/osLocale.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "osLocale.h"
+
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#if (_WIN64)
+#include <iphlpapi.h>
+#include <mswsock.h>
+#include <psapi.h>
+#include <stdio.h>
+#include <windows.h>
+#include <ws2tcpip.h>
+#pragma comment(lib, "Mswsock.lib ")
+#endif
+#include <objbase.h>
+#pragma warning(push)
+#pragma warning(disable : 4091)
+#include <DbgHelp.h>
+#pragma warning(pop)
+#elif defined(_TD_DARWIN_64)
+#include <errno.h>
+#include <libproc.h>
+#else
+#include <argp.h>
+#include <linux/sysctl.h>
+#include <sys/file.h>
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+#include <sys/syscall.h>
+#include <sys/utsname.h>
+#include <unistd.h>
+#endif
+
+typedef struct CharsetPair {
+  char *oldCharset;
+  char *newCharset;
+} CharsetPair;
+
+char *taosCharsetReplace(char *charsetstr) {
+  CharsetPair charsetRep[] = {
+      {"utf8", "UTF-8"},
+      {"936", "CP936"},
+  };
+
+  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
+    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
+      return strdup(charsetRep[i].newCharset);
+    }
+  }
+
+  return strdup(charsetstr);
+}
+
+/**
+ * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
+ * both the TDengine Server and the Client may be interrupted.
+ *
+ * In case that the setLocale failed to be executed, the right charset needs to be set.
+ */
+void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
+  char *locale = setlocale(LC_CTYPE, inLocale);
+
+  // default locale or user specified locale is not valid, abort launch
+  if (inLocale == NULL) {
+    printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale);
+  }
+
+  if (strlen(inCharSet) == 0) {
+    printf("failed to get charset, please set the valid charset in config file\n");
+    exit(-1);
+  }
+
+  if (!taosValidateEncodec(inCharSet)) {
+    printf("Invalid charset:%s, please set the valid charset in config file", inCharSet);
+    exit(-1);
+  }
+}
+
+void taosGetSystemLocale(char *outLocale, char *outCharset) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  char *locale = setlocale(LC_CTYPE, "chs");
+  if (locale != NULL) {
+    tstrncpy(outLocale, locale, TD_LOCALE_LEN);
+  }
+  strcpy(outCharset, "cp936");
+
+#elif defined(_TD_DARWIN_64)
+  /*
+   * originally from src/os/src/detail/osSysinfo.c
+   * POSIX format locale string:
+   * (Language Strings)_(Country/Region Strings).(code_page)
+   *
+   * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
+   *
+   * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
+   *
+   * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
+   * for libiconv that is employed to convert string in this system. This program will automatically use
+   * UTF-8 instead as the charset.
+   *
+   * In case of windows client, the locale string is not valid POSIX format, user needs to set the
+   * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
+   * CP936, CP437 for English charset.
+   *
+   */
+
+  char  sep = '.';
+  char *locale = NULL;
+
+  locale = setlocale(LC_CTYPE, "");
+  if (locale == NULL) {
+    // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
+    strcpy(outLocale, "en_US.UTF-8");
+  } else {
+    tstrncpy(outLocale, locale, TD_LOCALE_LEN);
+    // printf("locale not configured, set to system default:%s", outLocale);
+  }
+
+  /* if user does not specify the charset, extract it from locale */
+  char *str = strrchr(outLocale, sep);
+  if (str != NULL) {
+    str++;
+
+    char *revisedCharset = taosCharsetReplace(str);
+    tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN);
+
+    free(revisedCharset);
+    // printf("charset not configured, set to system default:%s", outCharset);
+  } else {
+    strcpy(outCharset, "UTF-8");
+    // printf("can't get locale and charset from system, set it to UTF-8");
+  }
+
+#else
+  /*
+   * POSIX format locale string:
+   * (Language Strings)_(Country/Region Strings).(code_page)
+   *
+   * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
+   *
+   * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
+   *
+   * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
+   * for libiconv that is employed to convert string in this system. This program will automatically use
+   * UTF-8 instead as the charset.
+   *
+   * In case of windows client, the locale string is not valid POSIX format, user needs to set the
+   * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
+   * CP936, CP437 for English charset.
+   *
+   */
+  char  sep = '.';
+  char *locale = NULL;
+
+  locale = setlocale(LC_CTYPE, "");
+  if (locale == NULL) {
+    // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
+    strcpy(outLocale, "en_US.UTF-8");
+  } else {
+    tstrncpy(outLocale, locale, TD_LOCALE_LEN);
+    // printf("locale not configured, set to system default:%s", outLocale);
+  }
+
+  // if user does not specify the charset, extract it from locale
+  char *str = strrchr(outLocale, sep);
+  if (str != NULL) {
+    str++;
+
+    char *revisedCharset = taosCharsetReplace(str);
+    tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN);
+
+    free(revisedCharset);
+    // printf("charset not configured, set to system default:%s", outCharset);
+  } else {
+    strcpy(outCharset, "UTF-8");
+    // printf("can't get locale and charset from system, set it to UTF-8");
+  }
+
+#endif
+}

source/os/src/osString.c

@@ -18,24 +18,6 @@
 #include <wchar.h>
 #include <wctype.h>
 
-typedef struct CharsetPair {
-  char *oldCharset;
-  char *newCharset;
-} CharsetPair;
-
-char *taosCharsetReplace(char *charsetstr) {
-  CharsetPair charsetRep[] = {
-      { "utf8", "UTF-8" }, { "936", "CP936" },
-  };
-
-  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
-    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
-      return strdup(charsetRep[i].newCharset);
-    }
-  }
-
-  return strdup(charsetstr);
-}
 
 int64_t taosStr2int64(const char *str) {
   char *endptr = NULL;

source/os/src/osSysinfo.c

@@ -21,9 +21,6 @@ int64_t tsPageSize = 0;
 int64_t tsOpenMax = 0;
 int64_t tsStreamMax = 0;
 int32_t tsNumOfCores = 1;
-char    tsLocale[TD_LOCALE_LEN] = {0};
-char    tsCharset[TD_LOCALE_LEN] = {0};  // default encode string
-
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 
 /*
@@ -89,17 +86,6 @@ bool taosGetProcMemory(float *memoryUsedMB) {
 }
 
 
-
-static void taosGetSystemLocale() {
-  // get and set default locale
-  char *locale = setlocale(LC_CTYPE, "chs");
-  if (locale != NULL) {
-    tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
-  }
-
-  strcpy(tsCharset, "cp936");
-}
-
 int32_t taosGetCpuCores() {
   SYSTEM_INFO info;
   GetSystemInfo(&info);
@@ -191,7 +177,6 @@ void taosGetSystemInfo() {
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
 
-  taosGetSystemLocale();
 }
 
 void taosKillSystem() {
@@ -263,52 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; }
 #include <errno.h>
 #include <libproc.h>
 
-/*
- * originally from src/os/src/detail/osSysinfo.c
- * POSIX format locale string:
- * (Language Strings)_(Country/Region Strings).(code_page)
- *
- * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
- *
- * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
- *
- * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
- * for libiconv that is employed to convert string in this system. This program will automatically use
- * UTF-8 instead as the charset.
- *
- * In case of windows client, the locale string is not valid POSIX format, user needs to set the
- * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
- * CP936, CP437 for English charset.
- *
- */
-static void taosGetSystemLocale() {  // get and set default locale
-  char  sep = '.';
-  char *locale = NULL;
-
-  locale = setlocale(LC_CTYPE, "");
-  if (locale == NULL) {
-    //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
-    strcpy(tsLocale, "en_US.UTF-8");
-  } else {
-    tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
-    //printf("locale not configured, set to system default:%s", tsLocale);
-  }
-
-  /* if user does not specify the charset, extract it from locale */
-  char *str = strrchr(tsLocale, sep);
-  if (str != NULL) {
-    str++;
-
-    char *revisedCharset = taosCharsetReplace(str);
-    tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
-
-    free(revisedCharset);
-    //printf("charset not configured, set to system default:%s", tsCharset);
-  } else {
-    strcpy(tsCharset, "UTF-8");
-    //printf("can't get locale and charset from system, set it to UTF-8");
-  }
-}
 
 void taosKillSystem() {
   //printf("function taosKillSystem, exit!");
@@ -325,8 +264,6 @@ void taosGetSystemInfo() {
   long page_size = sysconf(_SC_PAGESIZE);
   tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024);
   tsPageSize = page_size;
-
-  taosGetSystemLocale();
 }
 
 bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
@@ -563,51 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
   return true;
 }
 
-/*
- * POSIX format locale string:
- * (Language Strings)_(Country/Region Strings).(code_page)
- *
- * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
- *
- * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
- *
- * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
- * for libiconv that is employed to convert string in this system. This program will automatically use
- * UTF-8 instead as the charset.
- *
- * In case of windows client, the locale string is not valid POSIX format, user needs to set the
- * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
- * CP936, CP437 for English charset.
- *
- */
-static void taosGetSystemLocale() {  // get and set default locale
-  char  sep = '.';
-  char *locale = NULL;
-
-  locale = setlocale(LC_CTYPE, "");
-  if (locale == NULL) {
-    //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
-    strcpy(tsLocale, "en_US.UTF-8");
-  } else {
-    tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
-    //printf("locale not configured, set to system default:%s", tsLocale);
-  }
-
-  // if user does not specify the charset, extract it from locale
-  char *str = strrchr(tsLocale, sep);
-  if (str != NULL) {
-    str++;
-
-    char *revisedCharset = taosCharsetReplace(str);
-    tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
-
-    free(revisedCharset);
-    //printf("charset not configured, set to system default:%s", tsCharset);
-  } else {
-    strcpy(tsCharset, "UTF-8");
-    //printf("can't get locale and charset from system, set it to UTF-8");
-  }
-}
 
 int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
 
@@ -836,7 +728,6 @@ void taosGetSystemInfo() {
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
 
-  taosGetSystemLocale();
 }
 
 void taosKillSystem() {

source/os/src/osTimeZone1.c → source/os/src/osTimezone.c

@@ -45,7 +45,7 @@
 #include <unistd.h>
 #endif
 
-void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
+void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
 #ifdef WINDOWS
   char winStr[TD_LOCALE_LEN * 2];
   sprintf(winStr, "TZ=%s", inTimezone);
@@ -84,7 +84,7 @@ void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outD
   *outDaylight = daylight;
 }
 
-void osGetSystemTimezone(char *outTimezone) {
+void taosGetSystemTimezone(char *outTimezone) {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   char *tz = getenv("TZ");
   if (tz == NULL || strlen(tz) == 0) {