Merge pull request #7255 from taosdata/feature/td-5861

[TD-5861]<feature>:move log.dn to http plugins

src/common/inc/tglobal.h

@@ -41,6 +41,7 @@ extern char     tsArbitrator[];
 extern int8_t   tsArbOnline;
 extern int64_t  tsArbOnlineTimestamp;
 extern int32_t  tsDnodeId;
+extern int64_t  tsDnodeStartTime;
 
 // common
 extern int      tsRpcTimer;

src/common/src/tglobal.c

@@ -46,6 +46,7 @@ int8_t   tsArbOnline = 0;
 int64_t  tsArbOnlineTimestamp = TSDB_ARB_DUMMY_TIME;
 char     tsEmail[TSDB_FQDN_LEN] = {0};
 int32_t  tsDnodeId = 0;
+int64_t  tsDnodeStartTime = 0;
 
 // common
 int32_t tsRpcTimer       = 300;

src/dnode/src/dnodeMain.c

@@ -195,6 +195,7 @@ int32_t dnodeInitSystem() {
   dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
   moduleStart();
 
+  tsDnodeStartTime = taosGetTimestampMs();
   dnodeReportStep("TDengine", "initialized successfully", 1);
   dInfo("TDengine is initialized successfully");
 

src/inc/tfs.h

@@ -41,9 +41,16 @@ typedef struct {
   int64_t avail;
 } SFSMeta;
 
+typedef struct {
+  int64_t size;
+  int64_t used;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+
 int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
 void tfsDestroy();
-void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels);
 void tfsGetMeta(SFSMeta *pMeta);
 void tfsAllocDisk(int expLevel, int *level, int *id);
 

src/os/inc/osSysinfo.h

@@ -28,8 +28,11 @@ typedef struct {
 
 int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 
+int32_t taosGetCpuCores();
 void taosGetSystemInfo();
+bool taosReadProcIO(int64_t* rchars, int64_t* wchars);
 bool taosGetProcIO(float *readKB, float *writeKB);
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
 bool taosGetBandSpeed(float *bandSpeedKb);
 void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;

src/os/src/darwin/dwSysInfo.c

@@ -164,6 +164,10 @@ void taosKillSystem() {
   exit(0);
 }
 
+int32_t taosGetCpuCores() {
+  return sysconf(_SC_NPROCESSORS_ONLN);
+}
+
 void taosGetSystemInfo() {
   // taosGetProcInfos();
 
@@ -185,12 +189,25 @@ void taosGetSystemInfo() {
   taosGetSystemLocale();
 }
 
+bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
+  if (rchars) *rchars = 0;
+  if (wchars) *wchars = 0;
+  return true;
+}
+
 bool taosGetProcIO(float *readKB, float *writeKB) {
   *readKB = 0;
   *writeKB = 0;
   return true;
 }
 
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
+  if (bytes) *bytes = 0;
+  if (rbytes) *rbytes = 0;
+  if (tbytes) *tbytes = 0;
+  return true;
+}
+
 bool taosGetBandSpeed(float *bandSpeedKb) {
   *bandSpeedKb = 0;
   return true;

src/os/src/detail/osSysinfo.c

@@ -277,7 +277,7 @@ static void taosGetSystemLocale() {  // get and set default locale
   }
 }
 
-static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
+int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
 
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   static uint64_t lastSysUsed = 0;
@@ -332,7 +332,7 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   }
 }
 
-static bool taosGetCardInfo(int64_t *bytes) {
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
   *bytes = 0;
   FILE *fp = fopen(tsSysNetFile, "r");
   if (fp == NULL) {
@@ -347,9 +347,9 @@ static bool taosGetCardInfo(int64_t *bytes) {
   while (!feof(fp)) {
     memset(line, 0, len);
 
-    int64_t rbytes = 0;
+    int64_t o_rbytes = 0;
     int64_t rpackts = 0;
-    int64_t tbytes = 0;
+    int64_t o_tbytes = 0;
     int64_t tpackets = 0;
     int64_t nouse1 = 0;
     int64_t nouse2 = 0;
@@ -374,8 +374,10 @@ static bool taosGetCardInfo(int64_t *bytes) {
     sscanf(line,
            "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64
            " %" PRId64,
-           nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets);
-    *bytes += (rbytes + tbytes);
+           nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets);
+    if (rbytes) *rbytes = o_rbytes;
+    if (tbytes) *tbytes = o_tbytes;
+    *bytes += (o_rbytes + o_tbytes);
   }
 
   tfree(line);
@@ -390,7 +392,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
   int64_t        curBytes = 0;
   time_t         curTime = time(NULL);
 
-  if (!taosGetCardInfo(&curBytes)) {
+  if (!taosGetCardInfo(&curBytes, NULL, NULL)) {
     return false;
   }
 
@@ -420,7 +422,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) {
   return true;
 }
 
-static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
+bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
   FILE *fp = fopen(tsProcIOFile, "r");
   if (fp == NULL) {
     uError("open file:%s failed", tsProcIOFile);
@@ -441,10 +443,10 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
       break;
     }
     if (strstr(line, "rchar:") != NULL) {
-      sscanf(line, "%s %" PRId64, tmp, readbyte);
+      sscanf(line, "%s %" PRId64, tmp, rchars);
       readIndex++;
     } else if (strstr(line, "wchar:") != NULL) {
-      sscanf(line, "%s %" PRId64, tmp, writebyte);
+      sscanf(line, "%s %" PRId64, tmp, wchars);
       readIndex++;
     } else {
     }

src/os/src/windows/wSysinfo.c

@@ -115,7 +115,7 @@ static void taosGetSystemLocale() {
   }
 }
 
-static int32_t taosGetCpuCores() {
+int32_t taosGetCpuCores() {
   SYSTEM_INFO info;
   GetSystemInfo(&info);
   return (int32_t)info.dwNumberOfProcessors;
@@ -146,6 +146,13 @@ int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   }
 }
 
+bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) {
+  if (bytes) *bytes = 0;
+  if (rbytes) *rbytes = 0;
+  if (tbytes) *tbytes = 0;
+  return true;
+}
+
 bool taosGetBandSpeed(float *bandSpeedKb) {
   *bandSpeedKb = 0;
   return true;

src/plugins/http/inc/httpMetricsHandle.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_HTTPMETRICSHANDLE_H
+#define TDENGINE_HTTPMETRICSHANDLE_H
+
+#include "http.h"
+#include "httpInt.h"
+#include "httpUtil.h"
+#include "httpResp.h"
+
+void metricsInitHandle(HttpServer* httpServer);
+
+bool metricsProcessRequest(struct HttpContext* httpContext);
+
+#endif  // TDENGINE_HTTPMETRICHANDLE_H

src/plugins/http/src/httpMetricsHandle.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tfs.h"
+
+#include "httpMetricsHandle.h"
+#include "dnode.h"
+#include "httpLog.h"
+
+static HttpDecodeMethod metricsDecodeMethod = {"metrics", metricsProcessRequest};
+
+void metricsInitHandle(HttpServer* pServer) {
+  httpAddMethod(pServer, &metricsDecodeMethod);
+}
+
+bool metricsProcessRequest(HttpContext* pContext) {
+  httpDebug("context:%p, fd:%d, user:%s, process admin grant msg", pContext, pContext->fd, pContext->user);
+
+  JsonBuf* jsonBuf = httpMallocJsonBuf(pContext);
+  if (jsonBuf == NULL) {
+    httpError("failed to allocate memory for metrics");
+    httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
+    return false;
+  }
+
+  httpInitJsonBuf(jsonBuf, pContext);
+  httpWriteJsonBufHead(jsonBuf);
+
+  httpJsonToken(jsonBuf, JsonObjStt);
+  {
+    char* keyDisks = "tags";
+    httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks));
+    httpJsonToken(jsonBuf, JsonArrStt);
+    {
+      httpJsonItemToken(jsonBuf);
+      httpJsonToken(jsonBuf, JsonObjStt);
+      char* keyTagName = "name";
+      char* keyTagValue = "value";
+      httpJsonPairOriginString(jsonBuf, keyTagName, (int32_t)strlen(keyTagName), "\"dnode_id\"",
+                               (int32_t)strlen("\"dnode_id\""));
+      int32_t dnodeId = dnodeGetDnodeId();
+      httpJsonPairIntVal(jsonBuf, keyTagValue, (int32_t)strlen(keyTagValue), dnodeId);
+      httpJsonToken(jsonBuf, JsonObjEnd);
+    }
+    httpJsonToken(jsonBuf, JsonArrEnd);
+  }
+
+  {
+     if (tsDnodeStartTime != 0) {
+       int64_t now = taosGetTimestampMs();
+       int64_t upTime = now-tsDnodeStartTime;
+       char* keyUpTime = "up_time";
+       httpJsonPairInt64Val(jsonBuf, keyUpTime, (int32_t)strlen(keyUpTime), upTime);
+     }
+  }
+
+  {
+    int32_t cpuCores = taosGetCpuCores();
+    char* keyCpuCores = "cpu_cores";
+    httpJsonPairIntVal(jsonBuf, keyCpuCores, (int32_t)strlen(keyCpuCores), cpuCores);
+
+    float sysCpuUsage = 0;
+    float procCpuUsage = 0;
+    bool  succeeded = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
+    if (!succeeded) {
+      httpError("failed to get cpu usage");
+    } else {
+      if (sysCpuUsage <= procCpuUsage) {
+        sysCpuUsage = procCpuUsage + 0.1f;
+      }
+      char* keyCpuSystem = "cpu_system";
+      char* keyCpuEngine = "cpu_engine";
+      httpJsonPairFloatVal(jsonBuf, keyCpuSystem, (int32_t)strlen(keyCpuSystem), sysCpuUsage);
+      httpJsonPairFloatVal(jsonBuf, keyCpuEngine, (int32_t)strlen(keyCpuEngine), procCpuUsage);
+    }
+  }
+
+  {
+    float sysMemoryUsedMB = 0;
+    bool succeeded = taosGetSysMemory(&sysMemoryUsedMB);
+    if (!succeeded) {
+      httpError("failed to get sys memory info");
+    } else {
+      char* keyMemSystem = "mem_system";
+      httpJsonPairFloatVal(jsonBuf, keyMemSystem, (int32_t)strlen(keyMemSystem), sysMemoryUsedMB);
+    }
+
+    float procMemoryUsedMB = 0;
+    succeeded = taosGetProcMemory(&procMemoryUsedMB);
+    if (!succeeded) {
+      httpError("failed to get proc memory info");
+    } else {
+      char* keyMemEngine = "mem_engine";
+      httpJsonPairFloatVal(jsonBuf, keyMemEngine, (int32_t)strlen(keyMemEngine), procMemoryUsedMB);
+    }
+  }
+
+  {
+    int64_t bytes = 0, rbytes = 0, tbytes = 0;
+    bool succeeded = taosGetCardInfo(&bytes, &rbytes, &tbytes);
+    if (!succeeded) {
+      httpError("failed to get network info");
+    } else {
+      char* keyNetIn = "net_in";
+      char* keyNetOut = "net_out";
+      httpJsonPairInt64Val(jsonBuf, keyNetIn, (int32_t)strlen(keyNetIn), rbytes);
+      httpJsonPairInt64Val(jsonBuf, keyNetOut, (int32_t)strlen(keyNetOut), tbytes);
+    }
+  }
+
+  {
+    int64_t rchars = 0;
+    int64_t wchars = 0;
+    bool succeeded = taosReadProcIO(&rchars, &wchars);
+    if (!succeeded) {
+      httpError("failed to get io info");
+    } else {
+      char* keyIORead = "io_read";
+      char* keyIOWrite = "io_write";
+      httpJsonPairInt64Val(jsonBuf, keyIORead, (int32_t)strlen(keyIORead), rchars);
+      httpJsonPairInt64Val(jsonBuf, keyIOWrite, (int32_t)strlen(keyIOWrite), wchars);
+    }
+  }
+
+  {
+    const int8_t numTiers = 3;
+    SFSMeta      fsMeta;
+    STierMeta* tierMetas = calloc(numTiers, sizeof(STierMeta));
+    tfsUpdateInfo(&fsMeta, tierMetas, numTiers);
+    {
+      char* keyDiskUsed = "disk_used";
+      char* keyDiskTotal = "disk_total";
+      httpJsonPairInt64Val(jsonBuf, keyDiskTotal, (int32_t)strlen(keyDiskTotal), fsMeta.tsize);
+      httpJsonPairInt64Val(jsonBuf, keyDiskUsed, (int32_t)strlen(keyDiskUsed), fsMeta.used);
+      char* keyDisks = "disks";
+      httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks));
+      httpJsonToken(jsonBuf, JsonArrStt);
+      for (int i = 0; i < numTiers; ++i) {
+        httpJsonItemToken(jsonBuf);
+        httpJsonToken(jsonBuf, JsonObjStt);
+        char* keyDataDirLevelUsed = "datadir_used";
+        char* keyDataDirLevelTotal = "datadir_total";
+        httpJsonPairInt64Val(jsonBuf, keyDataDirLevelUsed, (int32_t)strlen(keyDataDirLevelUsed), tierMetas[i].used);
+        httpJsonPairInt64Val(jsonBuf, keyDataDirLevelTotal, (int32_t)strlen(keyDataDirLevelTotal), tierMetas[i].size);
+        httpJsonToken(jsonBuf, JsonObjEnd);
+      }
+      httpJsonToken(jsonBuf, JsonArrEnd);
+    }
+    free(tierMetas);
+  }
+
+  {
+    SStatisInfo info = dnodeGetStatisInfo();
+    {
+      char* keyReqHttp = "req_http";
+      char* keyReqSelect = "req_select";
+      char* keyReqInsert = "req_insert";
+      httpJsonPairInt64Val(jsonBuf, keyReqHttp, (int32_t)strlen(keyReqHttp), info.httpReqNum);
+      httpJsonPairInt64Val(jsonBuf, keyReqSelect, (int32_t)strlen(keyReqSelect), info.queryReqNum);
+      httpJsonPairInt64Val(jsonBuf, keyReqInsert, (int32_t)strlen(keyReqInsert), info.submitReqNum);
+    }
+  }
+
+  httpJsonToken(jsonBuf, JsonObjEnd);
+
+  httpWriteJsonBufEnd(jsonBuf);
+  pContext->reqType = HTTP_REQTYPE_OTHERS;
+  httpFreeJsonBuf(pContext);
+  return false;
+}
\ No newline at end of file

src/plugins/http/src/httpSystem.c

@@ -30,6 +30,7 @@
 #include "httpGcHandle.h"
 #include "httpRestHandle.h"
 #include "httpTgHandle.h"
+#include "httpMetricsHandle.h"
 
 #ifndef _ADMIN
 void adminInitHandle(HttpServer* pServer) {}
@@ -52,7 +53,7 @@ int32_t httpInitSystem() {
   gcInitHandle(&tsHttpServer);
   tgInitHandle(&tsHttpServer);
   opInitHandle(&tsHttpServer);
-
+  metricsInitHandle(&tsHttpServer);
   return 0;
 }
 

src/tfs/inc/tfsint.h

@@ -65,12 +65,7 @@ SDisk *tfsFreeDisk(SDisk *pDisk);
 int    tfsUpdateDiskInfo(SDisk *pDisk);
 
 // ttier.c ======================================================
-typedef struct {
-  int64_t size;
-  int64_t used;
-  int64_t free;
-  int16_t nAvailDisks;  // # of Available disks
-} STierMeta;
+
 typedef struct STier {
   pthread_spinlock_t lock;
   int                level;

src/tfs/src/tfs.c

@@ -101,7 +101,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
     return -1;
   }
 
-  tfsUpdateInfo(NULL);
+  tfsUpdateInfo(NULL, NULL, 0);
   for (int level = 0; level < TFS_NLEVEL(); level++) {
     tfsPosNextId(TFS_TIER_AT(level));
   }
@@ -119,7 +119,7 @@ void tfsDestroy() {
   }
 }
 
-void tfsUpdateInfo(SFSMeta *pFSMeta) {
+void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) {
   SFSMeta   fsMeta;
   STierMeta tierMeta;
 
@@ -130,11 +130,16 @@ void tfsUpdateInfo(SFSMeta *pFSMeta) {
   memset(pFSMeta, 0, sizeof(*pFSMeta));
 
   for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STierMeta *pTierMeta = &tierMeta;
+    if (tierMetas && level < numTiers) {
+      pTierMeta = tierMetas + level;
+    }
+
     STier *pTier = TFS_TIER_AT(level);
-    tfsUpdateTierInfo(pTier, &tierMeta);
-    pFSMeta->tsize += tierMeta.size;
-    pFSMeta->avail += tierMeta.free;
-    pFSMeta->used += tierMeta.used;
+    tfsUpdateTierInfo(pTier, pTierMeta);
+    pFSMeta->tsize += pTierMeta->size;
+    pFSMeta->avail += pTierMeta->free;
+    pFSMeta->used += pTierMeta->used;
   }
 
   tfsLock();
@@ -595,7 +600,7 @@ void taosGetDisk() {
   SFSMeta      fsMeta;
 
   if (tscEmbedded) {
-    tfsUpdateInfo(&fsMeta);
+    tfsUpdateInfo(&fsMeta, NULL, 0);
     tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
     tsUsedDataDirGB = (float)(fsMeta.used / unit);
     tsAvailDataDirGB = (float)(fsMeta.avail / unit);