s3/evict: fetch atime from stat file

include/os/osFile.h

@@ -76,7 +76,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
 
 int32_t taosUmaskFile(int32_t maskVal);
 
-int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
+int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
 int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
 int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
 bool    taosCheckExistFile(const char *pathname);

source/dnode/mgmt/mgmt_mnode/src/mmFile.c

@@ -46,7 +46,7 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
     if (code < 0) return -1;
     tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
     if (code < 0) return -1;
-    if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){
+    if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
       pOption->numOfReplicas++;
     }
   }
@@ -65,7 +65,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
   char      file[PATH_MAX] = {0};
   snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     dInfo("mnode file:%s not exist", file);
     return 0;
   }

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -97,7 +97,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
   SWrapperCfg *pCfgs = NULL;
   snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     dInfo("vnode file:%s not exist", file);
     return 0;
   }

source/dnode/mgmt/node_util/src/dmEps.c

@@ -100,7 +100,7 @@ int32_t dmReadEps(SDnodeData *pData) {
     goto _OVER;
   }
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     dInfo("dnode file:%s not exist", file);
     code = 0;
     goto _OVER;
@@ -350,7 +350,7 @@ void dmRotateMnodeEpSet(SDnodeData *pData) {
 }
 
 void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  if(!pData->validMnodeEps) return;
+  if (!pData->validMnodeEps) return;
   dmGetMnodeEpSet(pData, pEpSet);
   dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse);
   for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
@@ -469,7 +469,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
   char      file[PATH_MAX] = {0};
   snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     dDebug("dnode file:%s not exist", file);
     code = 0;
     goto _OVER;

source/dnode/mgmt/node_util/src/dmFile.c

@@ -38,7 +38,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
   char      file[PATH_MAX] = {0};
   snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     dInfo("file:%s not exist", file);
     code = 0;
     goto _OVER;

source/dnode/vnode/src/inc/vndCos.h

@@ -30,8 +30,9 @@ void    s3PutObjectFromFile(const char *file, const char *object);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);
 bool    s3Get(const char *object_name, const char *path);
-void    s3EvictCache();
+void    s3EvictCache(const char *path, long object_size);
 long    s3Size(const char *object_name);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/tq/tqOffsetSnapshot.c

@@ -60,7 +60,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
   }
 
   int64_t sz = 0;
-  if (taosStatFile(fname, &sz, NULL) < 0) {
+  if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
     taosCloseFile(&pFile);
     taosMemoryFree(fname);
     return -1;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
   // SDelFile
   if (pTsdb->fs.pDelFile) {
     tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
-    if (taosStatFile(fname, &size, NULL)) {
+    if (taosStatFile(fname, &size, NULL, NULL)) {
       code = TAOS_SYSTEM_ERROR(errno);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -195,7 +195,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
 
     // head =========
     tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
-    if (taosStatFile(fname, &size, NULL)) {
+    if (taosStatFile(fname, &size, NULL, NULL)) {
       code = TAOS_SYSTEM_ERROR(errno);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -206,7 +206,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
 
     // data =========
     tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
-    if (taosStatFile(fname, &size, NULL)) {
+    if (taosStatFile(fname, &size, NULL, NULL)) {
       code = TAOS_SYSTEM_ERROR(errno);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -221,7 +221,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
 
     // sma =============
     tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
-    if (taosStatFile(fname, &size, NULL)) {
+    if (taosStatFile(fname, &size, NULL, NULL)) {
       code = TAOS_SYSTEM_ERROR(errno);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -237,7 +237,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
     // stt ===========
     for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
       tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
-      if (taosStatFile(fname, &size, NULL)) {
+      if (taosStatFile(fname, &size, NULL, NULL)) {
         code = TAOS_SYSTEM_ERROR(errno);
         TSDB_CHECK_CODE(code, lino, _exit);
       }

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -38,7 +38,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
     const char *object_name = taosDirEntryBaseName((char *)path);
     long        s3_size = s3Size(object_name);
     if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
-      s3EvictCache();
+      s3EvictCache(path, s3_size);
       s3Get(object_name, path);
 
       pFD->pFD = taosOpenFile(path, flag);
@@ -66,7 +66,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
 
   // not check file size when reading data files.
   if (flag != TD_FILE_READ) {
-    if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
+    if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
       code = TAOS_SYSTEM_ERROR(errno);
       taosMemoryFree(pFD->pBuf);
       taosCloseFile(&pFD->pFD);

source/dnode/vnode/src/vnd/vnodeCos.c

@@ -190,7 +190,71 @@ bool s3Get(const char *object_name, const char *path) {
   return ret;
 }
 
-void s3EvictCache() {}
+typedef struct {
+  int64_t size;
+  int32_t atime;
+  char    name[TSDB_FILENAME_LEN];
+} SEvictFile;
+
+static int32_t evictFileCompareAsce(const void *pLeft, const void *pRight) {
+  SEvictFile *lhs = (SEvictFile *)pLeft;
+  SEvictFile *rhs = (SEvictFile *)pRight;
+  return lhs->atime < rhs->atime ? -1 : 1;
+}
+
+void s3EvictCache(const char *path, long object_size) {
+  SDiskSize disk_size = {0};
+  if (taosGetDiskSize((char *)path, &disk_size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    vError("failed to get disk:%s size since %s", path, terrstr());
+    return;
+  }
+
+  if (object_size >= disk_size.avail + 1 << 30) {
+    // evict too old files
+    // 1, list data files' atime under dir(path)
+    char dir_name[TSDB_FILENAME_LEN] = "\0";
+    tstrncpy(dir_name, path, TSDB_FILENAME_LEN);
+    taosDirName(dir_name);
+
+    tdbDirPtr pDir = taosOpenDir(dir_name);
+    if (pDir == NULL) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      vError("failed to open %s since %s", dir_name, terrstr());
+    }
+    SArray        *evict_files = taosArrayInit(16, sizeof(SEvictFile));
+    tdbDirEntryPtr pDirEntry;
+    while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+      char *name = taosGetDirEntryName(pDirEntry);
+      if (!strncmp(name + strlen(name) - 5, ".data", 5)) {
+        SEvictFile e_file = {0};
+
+        tstrncpy(e_file.name, name, TSDB_FILENAME_LEN);
+        taosStatFile(name, &e_file.size, NULL, &e_file.atime);
+
+        taosArrayPush(evict_files, &e_file);
+      }
+    }
+    taosCloseDir(&pDir);
+
+    // 2, sort by atime
+    taosArraySort(evict_files, evictFileCompareAsce);
+
+    // 3, remove files ascendingly until we get enough object_size space
+    long   evict_size = 0;
+    size_t ef_size = TARRAY_SIZE(evict_files);
+    for (size_t i = 0; i < ef_size; ++i) {
+      SEvictFile *evict_file = taosArrayGet(evict_files, i);
+      taosRemoveFile(evict_file->name);
+      evict_size += evict_file->size;
+      if (evict_size >= object_size) {
+        break;
+      }
+    }
+
+    taosArrayDestroy(evict_files);
+  }
+}
 
 long s3Size(const char *object_name) {
   long size = 0;

source/libs/function/src/udfd.c

@@ -378,9 +378,9 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
                                                  "pyUdfDestroy",   "pyUdfScalarProc", "pyUdfAggStart",
                                                  "pyUdfAggFinish", "pyUdfAggProc",    "pyUdfAggMerge"};
   void      **funcs[UDFD_MAX_PLUGIN_FUNCS] = {
-      (void **)&plugin->openFunc,         (void **)&plugin->closeFunc,         (void **)&plugin->udfInitFunc,
-      (void **)&plugin->udfDestroyFunc,   (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc,
-      (void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc,    (void **)&plugin->udfAggMergeFunc};
+           (void **)&plugin->openFunc,         (void **)&plugin->closeFunc,         (void **)&plugin->udfInitFunc,
+           (void **)&plugin->udfDestroyFunc,   (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc,
+           (void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc,    (void **)&plugin->udfAggMergeFunc};
   int32_t err = udfdLoadSharedLib(plugin->libPath, &plugin->lib, funcName, funcs, UDFD_MAX_PLUGIN_FUNCS);
   if (err != 0) {
     fnError("can not load python plugin. lib path %s", plugin->libPath);
@@ -848,7 +848,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
 
   char path[PATH_MAX] = {0};
   udfdGetFuncBodyPath(udf, path);
-  bool fileExist = !(taosStatFile(path, NULL, NULL) < 0);
+  bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
   if (fileExist) {
     strncpy(udf->path, path, PATH_MAX);
     fnInfo("udfd func body file. reuse existing file %s", path);

source/libs/index/src/indexFstFile.c

@@ -162,7 +162,7 @@ static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
       return ctx->offset;
     } else {
       int64_t file_size = 0;
-      taosStatFile(ctx->file.buf, &file_size, NULL);
+      taosStatFile(ctx->file.buf, &file_size, NULL, NULL);
       return (int)file_size;
     }
   }
@@ -199,7 +199,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
       code = taosFtruncateFile(ctx->file.pFile, 0);
       UNUSED(code);
 
-      code = taosStatFile(path, &ctx->file.size, NULL);
+      code = taosStatFile(path, &ctx->file.size, NULL, NULL);
       UNUSED(code);
 
       ctx->file.wBufOffset = 0;

source/libs/sync/src/syncRaftStore.c

@@ -42,7 +42,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) {
   const char *file = pNode->raftStorePath;
   SRaftStore *pStore = &pNode->raftStore;
 
-  if (taosStatFile(file, NULL, NULL) < 0) {
+  if (taosStatFile(file, NULL, NULL, NULL) < 0) {
     sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file);
     pStore->currentTerm = 0;
     pStore->voteFor.addr = 0;

source/libs/wal/src/walMeta.c

@@ -53,7 +53,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
   walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
 
   int64_t fileSize = 0;
-  taosStatFile(fnameStr, &fileSize, NULL);
+  taosStatFile(fnameStr, &fileSize, NULL, NULL);
 
   TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
   if (pFile == NULL) {
@@ -304,7 +304,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
 
     walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
     int32_t mtime = 0;
-    if (taosStatFile(fnameStr, NULL, &mtime) < 0) {
+    if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
       return -1;
@@ -353,7 +353,7 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
   walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
 
   int64_t fileSize = 0;
-  taosStatFile(fnameStr, &fileSize, NULL);
+  taosStatFile(fnameStr, &fileSize, NULL, NULL);
   int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
   int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
 
@@ -436,7 +436,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
     SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
 
     walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
-    int32_t code = taosStatFile(fnameStr, &fileSize, NULL);
+    int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
     if (code < 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
@@ -522,7 +522,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
   walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
   int64_t fileSize = 0;
 
-  if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) {
+  if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
     wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -935,7 +935,7 @@ int walLoadMeta(SWal* pWal) {
   walBuildMetaName(pWal, metaVer, fnameStr);
   // read metafile
   int64_t fileSize = 0;
-  taosStatFile(fnameStr, &fileSize, NULL);
+  taosStatFile(fnameStr, &fileSize, NULL, NULL);
   if (fileSize == 0) {
     (void)taosRemoveFile(fnameStr);
     wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);

source/os/src/osFile.c

@@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
 #endif
 }
 
-int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
+int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
 #ifdef WINDOWS
   struct _stati64 fileStat;
   int32_t         code = _stati64(path, &fileStat);
@@ -211,6 +211,10 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
     *mtime = fileStat.st_mtime;
   }
 
+  if (atime != NULL) {
+    *atime = fileStat.st_mtime;
+  }
+
   return 0;
 }
 int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
@@ -540,7 +544,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
 
 #ifdef WINDOWS
   struct __stat64 fileStat;
-  int32_t code = _fstat64(pFile->fd, &fileStat);
+  int32_t         code = _fstat64(pFile->fd, &fileStat);
 #else
   struct stat fileStat;
   int32_t code = fstat(pFile->fd, &fileStat);
@@ -897,17 +901,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
     goto cmp_end;
   }
 
-   dstFp = gzdopen(pFile->fd, "wb6f");
-   if (dstFp == NULL) {
-     ret = -3;
-     taosCloseFile(&pFile);
-     goto cmp_end;
-   }
+  dstFp = gzdopen(pFile->fd, "wb6f");
+  if (dstFp == NULL) {
+    ret = -3;
+    taosCloseFile(&pFile);
+    goto cmp_end;
+  }
 
-   while (!feof(pSrcFile->fp)) {
-     len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
-     (void)gzwrite(dstFp, data, len);
-   }
+  while (!feof(pSrcFile->fp)) {
+    len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
+    (void)gzwrite(dstFp, data, len);
+  }
 
 cmp_end:
   if (pSrcFile) {

source/util/src/tlog.c

@@ -17,8 +17,8 @@
 #include "tlog.h"
 #include "os.h"
 #include "tconfig.h"
-#include "tjson.h"
 #include "tglobal.h"
+#include "tjson.h"
 
 #define LOG_MAX_LINE_SIZE             (10024)
 #define LOG_MAX_LINE_BUFFER_SIZE      (LOG_MAX_LINE_SIZE + 3)
@@ -74,12 +74,12 @@ static SLogObj  tsLogObj = {.fileNum = 1};
 static int64_t  tsAsyncLogLostLines = 0;
 static int32_t  tsDaylightActive; /* Currently in daylight saving time. */
 
-bool    tsLogEmbedded = 0;
-bool    tsAsyncLog = true;
+bool tsLogEmbedded = 0;
+bool tsAsyncLog = true;
 #ifdef ASSERT_NOT_CORE
-bool    tsAssert = false;
+bool tsAssert = false;
 #else
-bool    tsAssert = true;
+bool tsAssert = true;
 #endif
 int32_t tsNumOfLogLines = 10000000;
 int32_t tsLogKeepDays = 0;
@@ -160,7 +160,7 @@ int32_t taosInitSlowLog() {
 
   tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE);
   if (tsLogObj.slowHandle == NULL) return -1;
-  
+
   taosUmaskFile(0);
   tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
   if (tsLogObj.slowHandle->pFile == NULL) {
@@ -403,13 +403,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
     strcpy(name, fn);
     strcat(name, ".0");
   }
-  bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0;
+  bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
 
   if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
     strcpy(name, fn);
     strcat(name, ".1");
   }
-  bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0;
+  bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
 
   // if none of the log files exist, open 0, if both exists, open the old one
   if (!log0Exist && !log1Exist) {
@@ -576,7 +576,7 @@ void taosPrintSlowLog(const char *format, ...) {
   } else {
     taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len);
   }
-      
+
   taosMemoryFree(buffer);
 }
 
@@ -769,12 +769,12 @@ static void taosWriteLog(SLogBuff *pLogBuf) {
 static void *taosAsyncOutputLog(void *param) {
   SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle;
   SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle;
-  
+
   setThreadName("log");
   int32_t count = 0;
   int32_t updateCron = 0;
   int32_t writeInterval = 0;
-  
+
   while (1) {
     writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval);
     count += writeInterval;
@@ -834,12 +834,12 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, const char
   return true;
 }
 
-void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) {
+void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo) {
   const char *flags = "UTL FATAL ";
   ELogLevel   level = DEBUG_FATAL;
   int32_t     dflag = 255;
-  char filepath[PATH_MAX] = {0};
-  TdFilePtr pFile = NULL;
+  char        filepath[PATH_MAX] = {0};
+  TdFilePtr   pFile = NULL;
 
   if (pMsg && msgLen > 0) {
     snprintf(filepath, sizeof(filepath), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType);
@@ -856,16 +856,16 @@ void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, vo
     int64_t writeSize = taosWriteFile(pFile, &msgLen, sizeof(msgLen));
     if (sizeof(msgLen) != writeSize) {
       taosUnLockFile(pFile);
-      taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", 
-          filepath, pFile, writeSize, sizeof(msgLen), terrstr());
+      taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
+                   filepath, pFile, writeSize, sizeof(msgLen), terrstr());
       goto _return;
     }
 
     writeSize = taosWriteFile(pFile, pMsg, msgLen);
     if (msgLen != writeSize) {
       taosUnLockFile(pFile);
-      taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", 
-          filepath, pFile, writeSize, msgLen, terrstr());
+      taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s",
+                   filepath, pFile, writeSize, msgLen, terrstr());
       goto _return;
     }
 
@@ -883,7 +883,7 @@ _return:
   taosPrintTrace(flags, level, dflag, 4);
 #elif !defined(WINDOWS)
   taosPrintLog(flags, level, dflag, "sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid,
-        taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
+               taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
   taosPrintTrace(flags, level, dflag, 3);
 #else
   taosPrintTrace(flags, level, dflag, 8);
@@ -892,17 +892,17 @@ _return:
   taosMemoryFree(pMsg);
 }
 
-void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd) {
+void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd) {
   const char *flags = "UTL FATAL ";
   ELogLevel   level = DEBUG_FATAL;
   int32_t     dflag = 255;
   TdFilePtr   pFile = NULL;
   bool        truncateFile = false;
-  char*       buf = NULL;
+  char       *buf = NULL;
 
   if (NULL == *pFd) {
     int64_t filesize = 0;
-    if (taosStatFile(filepath, &filesize, NULL) < 0) {
+    if (taosStatFile(filepath, &filesize, NULL, NULL) < 0) {
       if (ENOENT == errno) {
         return;
       }
@@ -916,7 +916,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
       return;
     }
 
-    pFile = taosOpenFile(filepath, TD_FILE_READ|TD_FILE_WRITE);
+    pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_WRITE);
     if (pFile == NULL) {
       if (ENOENT == errno) {
         return;
@@ -926,7 +926,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
       taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr());
       return;
     }
-    
+
     taosLockFile(pFile);
   } else {
     pFile = *pFd;
@@ -937,8 +937,8 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
   if (sizeof(msgLen) != readSize) {
     truncateFile = true;
     if (readSize < 0) {
-      taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", 
-          filepath, pFile, readSize, sizeof(msgLen), terrstr());
+      taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
+                   filepath, pFile, readSize, sizeof(msgLen), terrstr());
     }
     goto _return;
   }
@@ -948,12 +948,12 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
     taosPrintLog(flags, level, dflag, "failed to malloc buf, size:%" PRId64, msgLen);
     goto _return;
   }
-  
+
   readSize = taosReadFile(pFile, buf, msgLen);
   if (msgLen != readSize) {
     truncateFile = true;
-    taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", 
-        filepath, pFile, readSize, msgLen, terrstr());
+    taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", filepath,
+                 pFile, readSize, msgLen, terrstr());
     goto _return;
   }
 
@@ -981,7 +981,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile) {
   if (truncateFile) {
     taosFtruncateFile(pFile, 0);
   }
-  
+
   taosUnLockFile(pFile);
   taosCloseFile(&pFile);
 }

tools/shell/src/shellEngine.c

@@ -18,9 +18,9 @@
 #define _GNU_SOURCE
 #define _XOPEN_SOURCE
 #define _DEFAULT_SOURCE
-#include "shellInt.h"
-#include "shellAuto.h"
 #include "geosWrapper.h"
+#include "shellAuto.h"
+#include "shellInt.h"
 
 static bool    shellIsEmptyCommand(const char *cmd);
 static int32_t shellRunSingleCommand(char *command);
@@ -41,9 +41,9 @@ static bool    shellIsCommentLine(char *line);
 static void    shellSourceFile(const char *file);
 static void    shellGetGrantInfo();
 
-static void    shellCleanup(void *arg);
-static void   *shellCancelHandler(void *arg);
-static void   *shellThreadLoop(void *arg);
+static void  shellCleanup(void *arg);
+static void *shellCancelHandler(void *arg);
+static void *shellThreadLoop(void *arg);
 
 bool shellIsEmptyCommand(const char *cmd) {
   for (char c = *cmd++; c != 0; c = *cmd++) {
@@ -66,7 +66,7 @@ int32_t shellRunSingleCommand(char *command) {
   if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-result"
-          system("clear");
+    system("clear");
 #pragma GCC diagnostic pop
     return 0;
   }
@@ -142,8 +142,8 @@ int32_t shellRunCommand(char *command, bool recordHistory) {
     return 0;
   }
 
-  // add help or help; 
-  if(strncasecmp(command, "help;", 5) == 0) {
+  // add help or help;
+  if (strncasecmp(command, "help;", 5) == 0) {
     showHelp();
     return 0;
   }
@@ -223,14 +223,14 @@ void shellRunSingleCommandImp(char *command) {
   }
 
   // pre string
-  char * pre = "Query OK";
+  char *pre = "Query OK";
   if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) {
     pre = "Delete OK";
-  } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) {
+  } else if (shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) {
     pre = "Insert OK";
-  } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) {
+  } else if (shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) {
     pre = "Create OK";
-  } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) {
+  } else if (shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) {
     pre = "Drop OK";
   }
 
@@ -295,7 +295,7 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
   if (taosLocalTime(&tt, &ptm, buf) == NULL) {
     return buf;
   }
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
+  size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);
@@ -387,22 +387,20 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-    case TSDB_DATA_TYPE_JSON:
-      {
-        int32_t bufIndex = 0;
-        for (int32_t i = 0; i < length; i++) {
+    case TSDB_DATA_TYPE_JSON: {
+      int32_t bufIndex = 0;
+      for (int32_t i = 0; i < length; i++) {
+        buf[bufIndex] = val[i];
+        bufIndex++;
+        if (val[i] == '\"') {
           buf[bufIndex] = val[i];
           bufIndex++;
-          if (val[i] == '\"') {
-            buf[bufIndex] = val[i];
-            bufIndex++;
-          }
         }
-        buf[bufIndex] = 0;
-        
-        taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
       }
-      break;
+      buf[bufIndex] = 0;
+
+      taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
+    } break;
     case TSDB_DATA_TYPE_GEOMETRY:
       shellDumpHexValue(buf, val, length);
       taosFprintfFile(pFile, "%s", buf);
@@ -535,12 +533,10 @@ void shellPrintString(const char *str, int32_t width) {
 
   if (width == 0) {
     printf("%s", str);
-  }
-  else if (len > width) {
+  } else if (len > width) {
     if (width <= 3) {
       printf("%.*s.", width - 1, str);
-    }
-    else {
+    } else {
       printf("%.*s...", width - 3, str);
     }
   } else {
@@ -549,7 +545,7 @@ void shellPrintString(const char *str, int32_t width) {
 }
 
 void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) {
-  if (length == 0) {  //empty value
+  if (length == 0) {  // empty value
     shellPrintString("", width);
     return;
   }
@@ -565,7 +561,7 @@ void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width)
   char *outputWKT = NULL;
   code = doAsText(val, length, &outputWKT);
   if (code != TSDB_CODE_SUCCESS) {
-    shellPrintString(getThreadLocalGeosCtx()->errMsg, width);  //should NOT happen
+    shellPrintString(getThreadLocalGeosCtx()->errMsg, width);  // should NOT happen
     return;
   }
 
@@ -612,27 +608,26 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
       break;
     case TSDB_DATA_TYPE_FLOAT:
       if (tsEnableScience) {
-        printf("%*.7e",width,GET_FLOAT_VAL(val));
+        printf("%*.7e", width, GET_FLOAT_VAL(val));
       } else {
         n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
         if (n > SHELL_FLOAT_WIDTH) {
-
-            printf("%*.7e", width,GET_FLOAT_VAL(val));
+          printf("%*.7e", width, GET_FLOAT_VAL(val));
         } else {
-            printf("%s", buf);
+          printf("%s", buf);
         }
       }
       break;
     case TSDB_DATA_TYPE_DOUBLE:
       if (tsEnableScience) {
-        snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val));
+        snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
         printf("%s", buf);
       } else {
         n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
         if (n > SHELL_DOUBLE_WIDTH) {
-            printf("%*.15e", width, GET_DOUBLE_VAL(val));
+          printf("%*.15e", width, GET_DOUBLE_VAL(val));
         } else {
-            printf("%*s", width,buf);
+          printf("%*s", width, buf);
         }
       }
       break;
@@ -905,7 +900,7 @@ void shellReadHistory() {
   TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM);
   if (pFile == NULL) return;
 
-  char    *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
+  char   *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
   int32_t read_size = 0;
   while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
     line[read_size - 1] = '\0';
@@ -922,8 +917,8 @@ void shellReadHistory() {
   taosMemoryFreeClear(line);
   taosCloseFile(&pFile);
   int64_t file_size;
-  if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
-    TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
+  if (taosStatFile(pHistory->file, &file_size, NULL, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
+    TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
     if (pFile == NULL) return;
     int32_t endIndex = pHistory->hstart;
     if (endIndex != 0) {
@@ -945,7 +940,7 @@ void shellReadHistory() {
 void shellWriteHistory() {
   SShellHistory *pHistory = &shell.history;
   if (pHistory->hend == pHistory->hstart) return;
-  TdFilePtr      pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
+  TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
   if (pFile == NULL) return;
 
   for (int32_t i = pHistory->hstart; i != pHistory->hend;) {
@@ -991,7 +986,7 @@ void shellSourceFile(const char *file) {
     tstrncpy(fullname, file, PATH_MAX);
   }
 
-  sprintf(sourceFileCommand, "source %s;",fullname);
+  sprintf(sourceFileCommand, "source %s;", fullname);
   shellRecordCommandToHistory(sourceFileCommand);
 
   TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM);
@@ -1001,7 +996,7 @@ void shellSourceFile(const char *file) {
     return;
   }
 
-  char   *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
+  char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
   while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
     if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
     line[--read_len] = '\0';
@@ -1044,7 +1039,8 @@ void shellGetGrantInfo() {
 
   int32_t code = taos_errno(tres);
   if (code != TSDB_CODE_SUCCESS) {
-    if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS && code != TSDB_CODE_PAR_PERMISSION_DENIED) {
+    if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS &&
+        code != TSDB_CODE_PAR_PERMISSION_DENIED) {
       fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres));
     }
     return;
@@ -1080,7 +1076,8 @@ void shellGetGrantInfo() {
     } else if (strcmp(expiretime, "unlimited") == 0) {
       fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo);
     } else {
-      fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo, expiretime);
+      fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo,
+              expiretime);
     }
 
     taos_free_result(tres);
@@ -1123,9 +1120,9 @@ void *shellCancelHandler(void *arg) {
 #ifdef WEBSOCKET
     }
 #endif
-  #ifdef WINDOWS
+#ifdef WINDOWS
     printf("\n%s", shell.info.promptHeader);
-  #endif
+#endif
   }
 
   return NULL;
@@ -1165,8 +1162,7 @@ void *shellThreadLoop(void *arg) {
 }
 
 int32_t shellExecute() {
-  printf(shell.info.clientVersion, shell.info.cusName,
-         taos_get_client_info(), shell.info.cusName);
+  printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName);
   fflush(stdout);
 
   SShellArgs *pArgs = &shell.args;
@@ -1233,13 +1229,13 @@ int32_t shellExecute() {
   taosSetSignal(SIGTERM, shellQueryInterruptHandler);
   taosSetSignal(SIGHUP, shellQueryInterruptHandler);
   taosSetSignal(SIGINT, shellQueryInterruptHandler);
-  
+
 #ifdef WEBSOCKET
   if (!shell.args.restful && !shell.args.cloud) {
 #endif
 #ifndef WINDOWS
     printfIntroduction();
-#endif  
+#endif
     shellGetGrantInfo();
 #ifdef WEBSOCKET
   }

utils/test/c/tmqDemo.c

@@ -221,7 +221,7 @@ int64_t getDirectorySize(char* dir) {
       totalSize += subDirSize;
     } else if (0 == strcmp(strchr(fileName, '.'), ".log")) {  // only calc .log file size, and not include .idx file
       int64_t file_size = 0;
-      taosStatFile(subdir, &file_size, NULL);
+      taosStatFile(subdir, &file_size, NULL, NULL);
       totalSize += file_size;
     }
   }
@@ -702,4 +702,3 @@ int main(int32_t argc, char* argv[]) {
   taosCloseFile(&g_fp);
   return 0;
 }
-