[TD-5861]<feature>:move log.dn to http plugins

src/inc/tfs.h

@@ -41,9 +41,16 @@ typedef struct {
   int64_t avail;
 } SFSMeta;
 
+typedef struct {
+  int64_t size;
+  int64_t used;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+
 int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
 void tfsDestroy();
-void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
 void tfsGetMeta(SFSMeta *pMeta);
 void tfsAllocDisk(int expLevel, int *level, int *id);
 

src/os/inc/osSysinfo.h

@@ -28,8 +28,11 @@ typedef struct {
 
 int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 
+int32_t taosGetCpuCores();
 void taosGetSystemInfo();
+bool taosReadProcIO(int64_t* rchars, int64_t* wchars);
 bool taosGetProcIO(float *readKB, float *writeKB);
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
 bool taosGetBandSpeed(float *bandSpeedKb);
 void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;

src/os/src/detail/osSysinfo.c

@@ -277,7 +277,7 @@ static void taosGetSystemLocale() {  // get and set default locale
   }
 }
 
-static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
+int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
 
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   static uint64_t lastSysUsed = 0;
@@ -332,7 +332,7 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   }
 }
 
-static bool taosGetCardInfo(int64_t *bytes) {
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
   *bytes = 0;
   FILE *fp = fopen(tsSysNetFile, "r");
   if (fp == NULL) {
@@ -347,9 +347,9 @@ static bool taosGetCardInfo(int64_t *bytes) {
   while (!feof(fp)) {
     memset(line, 0, len);
 
-    int64_t rbytes = 0;
+    int64_t o_rbytes = 0;
     int64_t rpackts = 0;
-    int64_t tbytes = 0;
+    int64_t o_tbytes = 0;
     int64_t tpackets = 0;
     int64_t nouse1 = 0;
     int64_t nouse2 = 0;
@@ -374,8 +374,10 @@ static bool taosGetCardInfo(int64_t *bytes) {
     sscanf(line,
            "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64
            " %" PRId64,
-           nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
-    *bytes += (rbytes + tbytes);
+           nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets);
+    if (rbytes) *rbytes = o_rbytes;
+    if (tbytes) *tbytes = o_tbytes;
+    *bytes += (o_rbytes + o_tbytes);
   }
 
   tfree(line);
@@ -390,7 +392,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
   int64_t        curBytes = 0;
   time_t         curTime = time(NULL);
 
-  if (!taosGetCardInfo(&curBytes)) {
+  if (!taosGetCardInfo(&curBytes, NULL, NULL)) {
     return false;
   }
 
@@ -420,7 +422,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
   return true;
 }
 
-static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
+bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
   FILE *fp = fopen(tsProcIOFile, "r");
   if (fp == NULL) {
     uError("open file:%s failed", tsProcIOFile);
@@ -441,10 +443,10 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
       break;
     }
     if (strstr(line, "rchar:") != NULL) {
-      sscanf(line, "%s %" PRId64, tmp, readbyte);
+      sscanf(line, "%s %" PRId64, tmp, rchars);
       readIndex++;
     } else if (strstr(line, "wchar:") != NULL) {
-      sscanf(line, "%s %" PRId64, tmp, writebyte);
+      sscanf(line, "%s %" PRId64, tmp, wchars);
       readIndex++;
     } else {
     }

src/plugins/http/inc/httpMetricsHandle.h

+//
+// Created by slzhou on 8/6/21.
+//
+
+#ifndef TDENGINE_HTTPMETRICSHANDLE_H
+#define TDENGINE_HTTPMETRICSHANDLE_H
+
+#include "http.h"
+#include "httpInt.h"
+#include "httpUtil.h"
+#include "httpResp.h"
+
+void metricsInitHandle(HttpServer* httpServer);
+
+bool metricsProcessRequest(struct HttpContext* httpContext);
+
+#endif  // TDENGINE_HTTPMETRICHANDLE_H

src/plugins/http/src/httpMetricsHandle.c

+//
+// Created by slzhou on 8/6/21.
+//
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tfs.h"
+
+#include "httpMetricsHandle.h"
+#include "dnode.h"
+#include "httpLog.h"
+
+static HttpDecodeMethod metricsDecodeMethod = {"metrics", metricsProcessRequest};
+
+void metricsInitHandle(HttpServer* pServer) {
+  httpAddMethod(pServer, &metricsDecodeMethod);
+}
+
+bool metricsProcessRequest(HttpContext* pContext) {
+  httpDebug("context:%p, fd:%d, user:%s, process admin grant msg", pContext, pContext->fd, pContext->user);
+
+  JsonBuf* jsonBuf = httpMallocJsonBuf(pContext);
+  if (jsonBuf == NULL) {
+    httpError("failed to allocate memory for metrics");
+    httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
+    return false;
+  }
+
+  httpInitJsonBuf(jsonBuf, pContext);
+  httpWriteJsonBufHead(jsonBuf);
+
+  httpJsonToken(jsonBuf, JsonObjStt);
+  {
+    int32_t cpuCores = taosGetCpuCores();
+    char* keyCpuCores = "cpu_cores";
+    httpJsonPairIntVal(jsonBuf, keyCpuCores, strlen(keyCpuCores), cpuCores);
+
+    float sysCpuUsage = 0;
+    float procCpuUsage = 0;
+    bool  succeeded = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
+    if (!succeeded) {
+      httpError("failed to get cpu usage");
+    } else {
+      if (sysCpuUsage <= procCpuUsage) {
+        sysCpuUsage = procCpuUsage + 0.1f;
+      }
+      char* keyCpuSystem = "cpu_system";
+      char* keyCpuEngine = "cpu_engine";
+      httpJsonPairFloatVal(jsonBuf, keyCpuSystem, strlen(keyCpuSystem), sysCpuUsage);
+      httpJsonPairFloatVal(jsonBuf, keyCpuEngine, strlen(keyCpuEngine), procCpuUsage);
+    }
+  }
+
+  {
+    float sysMemoryUsedMB = 0;
+    bool succeeded = taosGetSysMemory(&sysMemoryUsedMB);
+    if (!succeeded) {
+      httpError("failed to get sys memory info");
+    } else {
+      char* keyMemSystem = "mem_system";
+      httpJsonPairFloatVal(jsonBuf, keyMemSystem, strlen(keyMemSystem), sysMemoryUsedMB);
+    }
+
+    float procMemoryUsedMB = 0;
+    succeeded = taosGetProcMemory(&procMemoryUsedMB);
+    if (!succeeded) {
+      httpError("failed to get proc memory info");
+    } else {
+      char* keyMemEngine = "mem_engine";
+      httpJsonPairFloatVal(jsonBuf, keyMemEngine, strlen(keyMemEngine), procMemoryUsedMB);
+    }
+  }
+
+  {
+    int64_t bytes = 0, rbytes = 0, tbytes = 0;
+    bool succeeded = taosGetCardInfo(&bytes, &rbytes, &tbytes);
+    if (!succeeded) {
+      httpError("failed to get network info");
+    } else {
+      char* keyNetIn = "net_in";
+      char* keyNetOut = "net_out";
+      httpJsonPairInt64Val(jsonBuf, keyNetIn, strlen(keyNetIn), rbytes);
+      httpJsonPairInt64Val(jsonBuf, keyNetOut, strlen(keyNetOut), tbytes);
+    }
+  }
+
+  {
+    int64_t rchars = 0;
+    int64_t wchars = 0;
+    bool succeeded = taosReadProcIO(&rchars, &wchars);
+    if (!succeeded) {
+      httpError("failed to get io info");
+    } else {
+      char* keyIORead = "io_read";
+      char* keyIOWrite = "io_write";
+      httpJsonPairInt64Val(jsonBuf, keyIORead, strlen(keyIORead), rchars);
+      httpJsonPairInt64Val(jsonBuf, keyIOWrite, strlen(keyIOWrite), wchars);
+    }
+  }
+
+  {
+    const int8_t numTiers = 3;
+    SFSMeta      fsMeta;
+    STierMeta    tierMetas[numTiers];
+    memset(tierMetas, 0, 3 * sizeof(STierMeta));
+    tfsUpdateInfo(&fsMeta, tierMetas, numTiers);
+    {
+      char* keyDiskUsed = "disk_used";
+      char* keyDiskTotal = "disk_total";
+      httpJsonPairInt64Val(jsonBuf, keyDiskTotal, strlen(keyDiskTotal), fsMeta.tsize);
+      httpJsonPairInt64Val(jsonBuf, keyDiskUsed, strlen(keyDiskUsed), fsMeta.used);
+      char* keyDisks = "disks";
+      httpJsonPairHead(jsonBuf, keyDisks, strlen(keyDisks));
+      httpJsonToken(jsonBuf, JsonArrStt);
+      for (int i = 0; i < numTiers; ++i) {
+        httpJsonItemToken(jsonBuf);
+        httpJsonToken(jsonBuf, JsonObjStt);
+        char* keyDataDirLevelUsed = "datadir_used";
+        char* keyDataDirLevelTotal = "datadir_total";
+        httpJsonPairInt64Val(jsonBuf, keyDataDirLevelUsed, strlen(keyDataDirLevelUsed), tierMetas[i].used);
+        httpJsonPairInt64Val(jsonBuf, keyDataDirLevelTotal, strlen(keyDataDirLevelTotal), tierMetas[i].size);
+        httpJsonToken(jsonBuf, JsonObjEnd);
+      }
+      httpJsonToken(jsonBuf, JsonArrEnd);
+    }
+  }
+
+  {
+    SStatisInfo info = dnodeGetStatisInfo();
+    {
+      char* keyReqHttp = "req_http";
+      char* keyReqSelect = "req_select";
+      char* keyReqInsert = "req_insert";
+      httpJsonPairInt64Val(jsonBuf, keyReqHttp, strlen(keyReqHttp), info.httpReqNum);
+      httpJsonPairInt64Val(jsonBuf, keyReqSelect, strlen(keyReqSelect), info.queryReqNum);
+      httpJsonPairInt64Val(jsonBuf, keyReqInsert, strlen(keyReqInsert), info.submitReqNum);
+    }
+  }
+
+  httpJsonToken(jsonBuf, JsonObjEnd);
+
+  httpWriteJsonBufEnd(jsonBuf);
+  pContext->reqType = HTTP_REQTYPE_OTHERS;
+  httpFreeJsonBuf(pContext);
+  return false;
+}
\ No newline at end of file

src/plugins/http/src/httpSystem.c

@@ -30,6 +30,7 @@
 #include "httpGcHandle.h"
 #include "httpRestHandle.h"
 #include "httpTgHandle.h"
+#include "httpMetricsHandle.h"
 
 #ifndef _ADMIN
 void adminInitHandle(HttpServer* pServer) {}
@@ -52,7 +53,7 @@ int32_t httpInitSystem() {
   gcInitHandle(&tsHttpServer);
   tgInitHandle(&tsHttpServer);
   opInitHandle(&tsHttpServer);
-
+  metricsInitHandle(&tsHttpServer);
   return 0;
 }
 

src/tfs/inc/tfsint.h

@@ -65,12 +65,7 @@ SDisk *tfsFreeDisk(SDisk *pDisk);
 int    tfsUpdateDiskInfo(SDisk *pDisk);
 
 // ttier.c ======================================================
-typedef struct {
-  int64_t size;
-  int64_t used;
-  int64_t free;
-  int16_t nAvailDisks;  // # of Available disks
-} STierMeta;
+
 typedef struct STier {
   pthread_spinlock_t lock;
   int                level;

src/tfs/src/tfs.c

@@ -101,7 +101,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
     return -1;
   }
 
-  tfsUpdateInfo(NULL);
+  tfsUpdateInfo(NULL, NULL, 0);
   for (int level = 0; level < TFS_NLEVEL(); level++) {
     tfsPosNextId(TFS_TIER_AT(level));
   }
@@ -119,7 +119,7 @@ void tfsDestroy() {
   }
 }
 
-void tfsUpdateInfo(SFSMeta *pFSMeta) {
+void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
   SFSMeta   fsMeta;
   STierMeta tierMeta;
 
@@ -130,11 +130,16 @@ void tfsUpdateInfo(SFSMeta *pFSMeta) {
   memset(pFSMeta, 0, sizeof(*pFSMeta));
 
   for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STierMeta *pTierMeta = &tierMeta;
+    if (tierMetas && level < numTiers) {
+      pTierMeta = tierMetas + level;
+    }
+
     STier *pTier = TFS_TIER_AT(level);
-    tfsUpdateTierInfo(pTier, &tierMeta);
-    pFSMeta->tsize += tierMeta.size;
-    pFSMeta->avail += tierMeta.free;
-    pFSMeta->used += tierMeta.used;
+    tfsUpdateTierInfo(pTier, pTierMeta);
+    pFSMeta->tsize += pTierMeta->size;
+    pFSMeta->avail += pTierMeta->free;
+    pFSMeta->used += pTierMeta->used;
   }
 
   tfsLock();
@@ -595,7 +600,7 @@ void taosGetDisk() {
   SFSMeta      fsMeta;
 
   if (tscEmbedded) {
-    tfsUpdateInfo(&fsMeta);
+    tfsUpdateInfo(&fsMeta, NULL, 0);
     tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
     tsUsedDataDirGB = (float)(fsMeta.used / unit);
     tsAvailDataDirGB = (float)(fsMeta.avail / unit);