diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index c12726d2920f236593f6707c0071efc73eeea189..fdb846670656b189d12368c519348367ec4d2974 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -189,6 +189,9 @@ # max number of rows per log filters # numOfLogLines 10000000 +# time of keeping log files, days +# logKeepDays 0 + # enable/disable async log # asyncLog 1 diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index fedafe5b022bdf54fb8439a10c7d6355485fef93..77e8b76456b04301a1f556c69a4bae98970c1f37 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -158,6 +158,7 @@ extern char buildinfo[]; // log extern int32_t tsAsyncLog; extern int32_t tsNumOfLogLines; +extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 9683a635038dffa4b06df9fb90413673ac0017ad..96e8fb26c65bc0c13a03d3d9b9c064d57c0fb45e 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -254,7 +254,7 @@ bool taosCfgDynamicOptions(char *msg) { //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue; - int32_t cfgLen = strlen(cfg->option); + int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; *((int32_t *)cfg->ptr) = vint; @@ -1013,12 +1013,22 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsNumOfLogLines; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10000; + cfg.minValue = 1000; cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "logKeepDays"; + cfg.ptr = &tsLogKeepDays; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 365000; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; cfg.valType = TAOS_CFG_VTYPE_INT16; diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index e7dc04fd1532660cabfbe407c5a8045f010b2d91..17683743e3ee7ed11350c02e504cce9f379b73d9 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -24,6 +24,7 @@ extern "C" { void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); void taosRename(char* oldName, char *newName); +void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); #ifdef __cplusplus } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 224e41593dac4bb4111d656d6e84f7c1f59fd93f..d4f3d6d2afd94bcfd2527af9927bf188ba4bf0de 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -39,6 +39,7 @@ #include #include #include +#include #include "msvcProcess.h" #include "msvcDirect.h" #include "msvcFcntl.h" @@ -58,8 +59,6 @@ extern "C" { int32_t BUILDIN_CTZL(uint64_t val); int32_t BUILDIN_CTZ(uint32_t val); -#define TAOS_OS_FUNC_DIR - #define TAOS_OS_FUNC_FILE #define TAOS_OS_FUNC_FILE_ISREG #define TAOS_OS_FUNC_FILE_ISDIR diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 7a537cdfead3a15e202b8d7d2bc5d40cff5281bc..93651c78ef98cfbcb8bbdbb575b3fb952acab229 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -18,8 +18,6 @@ #include "tglobal.h" #include "tulog.h" -#ifndef TAOS_OS_FUNC_DIR - void taosRemoveDir(char *rootDir) { DIR *dir = opendir(rootDir); if (dir == NULL) return; @@ -51,18 +49,54 @@ int taosMkDir(const char *path, mode_t mode) { } void taosRename(char* oldName, char *newName) { - if (0 == tsEnableVnodeBak) { - uInfo("vnode backup not enabled"); - return; - } - // if newName in not empty, rename return fail. // the newName must be empty or does not exist if (rename(oldName, newName)) { - uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno)); + uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { - uInfo("%s is modify to %s success!", oldName, newName); + uInfo("successfully to rename file %s to %s", oldName, newName); } } -#endif +void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { + DIR *dir = opendir(rootDir); + if (dir == NULL) return; + + int64_t sec = taosGetTimestampSec(); + struct dirent *de = NULL; + + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + + char filename[1024]; + snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + if (de->d_type & DT_DIR) { + continue; + } else { + // struct stat fState; + // if (stat(fname, &fState) < 0) { + // continue; + // } + int32_t len = (int32_t)strlen(filename); + int64_t fileSec = 0; + for (int i = len - 1; i >= 0; i--) { + if (filename[i] == '.') { + fileSec = atoll(filename + i + 1); + break; + } + } + + if (fileSec <= 100) continue; + int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); + if (days > keepDays) { + (void)remove(filename); + uInfo("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); + } else { + uTrace("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); + } + } + } + + closedir(dir); + rmdir(rootDir); +} diff --git a/src/os/src/windows/wDir.c b/src/os/src/windows/wDir.c deleted file mode 100644 index c486cd0d4076cd3800882299ea5ac8607f964bbb..0000000000000000000000000000000000000000 --- a/src/os/src/windows/wDir.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tulog.h" - -void taosRemoveDir(char *rootDir) { - uError("%s not implemented yet", __FUNCTION__); -} - -int taosMkDir(const char *path, mode_t mode) { - uError("%s not implemented yet", __FUNCTION__); - return 0; -} - -void taosMvDir(char* destDir, char *srcDir) { - uError("%s not implemented yet", __FUNCTION__); -} diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 0d9a28e288d9ea5885769b4fa3537dd9ba689f6e..1fb235a00575529be660cea18c1401f1d075a49b 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -58,11 +58,20 @@ char *strsep(char **stringp, const char *delim) { char *getpass(const char *prefix) { static char passwd[TSDB_KEY_LEN] = {0}; - + memset(passwd, 0, TSDB_KEY_LEN); printf("%s", prefix); - scanf("%s", passwd); - char n = getchar(); + int32_t index = 0; + char ch; + while (index < TSDB_KEY_LEN) { + ch = getch(); + if (ch == '\n' || ch == '\r') { + break; + } else { + passwd[index++] = ch; + } + } + return passwd; } @@ -131,11 +140,11 @@ int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) { } -/* Copy memory to memory until the specified number of bytes -has been copied, return pointer to following byte. -Overlap is NOT handled correctly. */ -void *mempcpy(void *dest, const void *src, size_t len) { - return (char*)memcpy(dest, src, len) + len; +/* Copy memory to memory until the specified number of bytes +has been copied, return pointer to following byte. +Overlap is NOT handled correctly. */ +void *mempcpy(void *dest, const void *src, size_t len) { + return (char*)memcpy(dest, src, len) + len; } /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */ diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4110c566d2913321036416d96d07c36cbde15f43..c7cd919ee8d2155d8440f47d3afe3f96dcc8c4bc 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -576,5 +576,5 @@ static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) { } static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) { - return TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision); + return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision)); } \ No newline at end of file diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 875c597008f19f6114bb6f6dbff8071925480afc..0ec55841a060a49f4aa9e29981fa426e42d29d5c 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -270,7 +270,7 @@ void taosReadGlobalLogCfg() { } wordfree(&full_path); - taosReadLogOption("tsLogDir", tsLogDir); + taosReadLogOption("logDir", tsLogDir); sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); @@ -288,9 +288,9 @@ void taosReadGlobalLogCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + taosGetline(&line, &len, fp); line[len - 1] = 0; - + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 766301914a60f347eeb71937d0806602f690ce7c..a8587de767bc4093a935ca3082a6fe65bbe74e44 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -62,6 +62,7 @@ typedef struct { pthread_mutex_t logMutex; } SLogObj; +int32_t tsLogKeepDays = 0; int32_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; @@ -78,6 +79,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); +extern void taosPrintGlobalCfg(); static int32_t taosStartLog() { pthread_attr_t threadAttr; @@ -136,11 +138,24 @@ static void taosUnLockFile(int32_t fd) { } } +static void taosKeepOldLog(char *oldName) { + if (tsLogKeepDays <= 0) return; + + int64_t fileSec = taosGetTimestampSec(); + char fileName[LOG_FILE_NAME_LEN + 20]; + snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); + + taosRename(oldName, fileName); + taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays); +} + static void *taosThreadToOpenNewFile(void *param) { - char name[LOG_FILE_NAME_LEN + 20]; + char keepName[LOG_FILE_NAME_LEN + 20]; + sprintf(keepName, "%s.%d", tsLogObj.logName, tsLogObj.flag); tsLogObj.flag ^= 1; tsLogObj.lines = 0; + char name[LOG_FILE_NAME_LEN + 20]; sprintf(name, "%s.%d", tsLogObj.logName, tsLogObj.flag); umask(0); @@ -150,6 +165,7 @@ static void *taosThreadToOpenNewFile(void *param) { uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno)); return NULL; } + taosLockFile(fd); (void)lseek(fd, 0, SEEK_SET); @@ -157,9 +173,13 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->fd = fd; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - uInfo("new log file is opened!!!"); - taosCloseLogByFd(oldFd); + + uInfo(" new log file:%d is opened", tsLogObj.flag); + uInfo("=================================="); + taosPrintGlobalCfg(); + taosKeepOldLog(keepName); + return NULL; } @@ -264,20 +284,23 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { strcat(name, ".0"); } + if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { + strcpy(name, fn); + strcat(name, ".1"); + } + + bool log0Exist = stat(name, &logstat0) >= 0; + bool log1Exist = stat(name, &logstat1) >= 0; + // if none of the log files exist, open 0, if both exists, open the old one - if (stat(name, &logstat0) < 0) { + if (!log0Exist && !log1Exist) { + tsLogObj.flag = 0; + } else if (!log1Exist) { tsLogObj.flag = 0; + } else if (!log0Exist) { + tsLogObj.flag = 1; } else { - if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - - if (stat(name, &logstat1) < 0) { - tsLogObj.flag = 1; - } else { - tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; - } + tsLogObj.flag = (logstat0.st_mtime > logstat1.st_mtime) ? 0 : 1; } char fileName[LOG_FILE_NAME_LEN + 50] = "\0"; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 5a778156ff4fa88a798d260654edf04860d9692b..a463a2c90a7cf440d4e30735d469861d02ce3eac 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -382,7 +382,13 @@ void vnodeRelease(void *pVnodeRaw) { char newDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); - taosRename(rootDir, newDir); + + if (0 == tsEnableVnodeBak) { + vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); + } else { + taosRename(rootDir, newDir); + } + taosRemoveDir(rootDir); dnodeSendStatusMsgToMnode(); }