Merge pull request #753 from taosdata/feature/lihui

[TBASE-1123]

src/client/src/tscAsync.c

@@ -24,6 +24,7 @@
 #include "tsocket.h"
 #include "tsql.h"
 #include "tutil.h"
+#include "tnote.h"
 
 void tscProcessFetchRow(SSchedMsg *pMsg);
 void tscProcessAsyncRetrieve(void *param, TAOS_RES *tres, int numOfRows);
@@ -53,7 +54,9 @@ void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *,
     tscError("sql string too long");
     tscQueueAsyncError(fp, param);
     return;
-  }
+  }  
+
+  taosNotePrintTsc(sqlstr);
 
   SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj));
   if (pSql == NULL) {

src/client/src/tscSql.c

@@ -27,6 +27,7 @@
 #include "tsql.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "tnote.h"
 
 TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                        void *param, void **taos) {
@@ -274,6 +275,8 @@ int taos_query(TAOS *taos, const char *sqlstr) {
     return pRes->code;
   }
 
+  taosNotePrintTsc(sqlstr);
+
   void *sql = realloc(pSql->sqlstr, sqlLen + 1);
   if (sql == NULL) {
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;

src/client/src/tscSystem.c

@@ -45,6 +45,10 @@ extern int            tscEmbedded;
 int                   tscNumOfThreads;
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 
+extern int  tsTscEnableRecordSql;
+extern int  tsNumOfLogLines;
+void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
+
 void tscCheckDiskUsage(void *para, void *unused) {
   taosGetDisk();
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
@@ -85,6 +89,10 @@ void taos_init_imp() {
 
   taosSetCoreDump();
 
+  if (tsTscEnableRecordSql != 0) {
+    taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note");
+  }
+  
 #ifdef CLUSTER
   tscMgmtIpList.numOfIps = 2;
   strcpy(tscMgmtIpList.ipstr[0], tsMasterIp);
@@ -354,4 +362,4 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
 
   atomic_store_32(&lock, 0);
   return ret;
-}
\ No newline at end of file
+}

src/inc/tglobalcfg.h

@@ -150,6 +150,8 @@ extern int   tsHttpEnableRecordSql;
 extern int   tsTelegrafUseFieldNum;
 extern int   tsAdminRowLimit;
 
+extern int   tsTscEnableRecordSql;
+
 extern char tsMonitorDbName[];
 extern char tsInternalPass[];
 extern int  tsMonitorInterval;

src/inc/tnote.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TNOTE_H
+#define TDENGINE_TNOTE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "unistd.h"
+#include "os.h"
+#include "tutil.h"
+#include "tglobalcfg.h"
+
+#define MAX_NOTE_LINE_SIZE 66000
+#define NOTE_FILE_NAME_LEN 300
+  
+typedef struct _taosNoteInfo {
+  int  taosNoteFileNum ;
+  int  taosNoteMaxLines;
+  int  taosNoteLines;
+  char taosNoteName[NOTE_FILE_NAME_LEN];
+  int  taosNoteFlag;
+  int  taosNoteFd;
+  int  taosNoteOpenInProgress;
+  pthread_mutex_t taosNoteMutex;
+}taosNoteInfo;
+  
+void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...);
+
+extern taosNoteInfo  m_HttpNote;
+extern taosNoteInfo  m_TscNote;
+
+extern int   tsHttpEnableRecordSql;
+extern int   tsTscEnableRecordSql;
+
+#define taosNotePrintHttp(...)               \
+  if (tsHttpEnableRecordSql) {               \
+    taosNotePrint(&m_HttpNote, __VA_ARGS__); \
+  }
+    
+#define taosNotePrintTsc(...)                 \
+    if (tsTscEnableRecordSql) {               \
+      taosNotePrint(&m_TscNote, __VA_ARGS__); \
+    }
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/modules/http/src/httpSql.c

@@ -24,11 +24,11 @@
 #include "httpResp.h"
 #include "taos.h"
 #include "tsclient.h"
+#include "tnote.h"
 
 void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos);
 void httpProcessMultiSql(HttpContext *pContext);
-void taosNotePrint(const char * const format, ...);
 
 void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
   HttpContext *pContext = (HttpContext *)param;
@@ -165,7 +165,7 @@ void httpProcessMultiSql(HttpContext *pContext) {
   char *sql = httpGetCmdsString(pContext, cmd->sql);
   httpDump("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd,
            pContext->ipstr, pContext->user, multiCmds->pos, sql);
-  taosNotePrint(sql);
+  taosNotePrintHttp(sql);
   taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext);
 }
 
@@ -298,7 +298,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) {
 
   httpDump("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr,
            pContext->user, sql);
-  taosNotePrint(sql);
+  taosNotePrintHttp(sql);
   taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext);
 }
 

src/modules/http/src/httpSystem.c

@@ -42,7 +42,7 @@
 #endif
 
 static HttpServer *httpServer = NULL;
-void taosInitNote(int numOfNoteLines, int maxNotes);
+void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
 
 int httpInitSystem() {
   taos_init();
@@ -61,7 +61,7 @@ int httpInitSystem() {
   pthread_mutex_init(&httpServer->serverMutex, NULL);
 
   if (tsHttpEnableRecordSql != 0) {
-    taosInitNote(tsNumOfLogLines / 10, 1);
+    taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note");
   }
   restInitHandle(httpServer);
   adminInitHandle(httpServer);

src/util/src/tglobalcfg.c

@@ -162,6 +162,8 @@ int tsHttpEnableRecordSql = 0;
 int tsTelegrafUseFieldNum = 0;
 int tsAdminRowLimit = 10240;
 
+int   tsTscEnableRecordSql = 0;
+
 int tsRpcTimer = 300;
 int tsRpcMaxTime = 600;      // seconds;
 
@@ -767,6 +769,9 @@ static void doInitGlobalConfig() {
                      0, 255, 0,
                      TSDB_CFG_UTYPE_NONE);
 
+  tsInitConfigOption(cfg++, "tscEnableRecordSql", &tsTscEnableRecordSql, TSDB_CFG_VTYPE_INT,
+                     TSDB_CFG_CTYPE_B_CONFIG,
+                     1, 100000, 0, TSDB_CFG_UTYPE_NONE);
   // version info
   tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING,
                      TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT,

src/util/src/tnote.c

@@ -13,42 +13,50 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "tutil.h"
-#include "tglobalcfg.h"
-
-#define MAX_NOTE_LINE_SIZE 66000
-#define NOTE_FILE_NAME_LEN 300
-
-static int  taosNoteFileNum = 1;
-static int  taosNoteMaxLines = 0;
-static int  taosNoteLines = 0;
-static char taosNoteName[NOTE_FILE_NAME_LEN];
-static int  taosNoteFlag = 0;
-static int  taosNoteFd = -1;
-static int  taosNoteOpenInProgress = 0;
-static pthread_mutex_t taosNoteMutex;
-void taosNotePrint(const char * const format, ...);
-int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum);
-
-void taosInitNote(int numOfNoteLines, int maxNotes)
+#include "tnote.h"
+
+taosNoteInfo  m_HttpNote;
+taosNoteInfo  m_TscNote;
+
+int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote);
+
+void taosInitNote(int numOfNoteLines, int maxNotes, char* lable)
 {
+    taosNoteInfo * pNote = NULL;
     char temp[128] = { 0 };
-    sprintf(temp, "%s/taosnote", logDir);
-    if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes) < 0)
+
+    if (strcasecmp(lable, "http_note") == 0) {
+        pNote = &m_HttpNote;
+        sprintf(temp, "%s/httpnote", logDir);
+    } else if (strcasecmp(lable, "tsc_note") == 0) {
+        pNote = &m_TscNote;        
+        sprintf(temp, "%s/tscnote-%d", logDir, getpid());
+    } else {
+        return;
+    }
+
+    memset(pNote, 0, sizeof(taosNoteInfo));
+    pNote->taosNoteFileNum        = 1;
+    //pNote->taosNoteMaxLines       = 0;
+    //pNote->taosNoteLines          = 0;
+    //pNote->taosNoteFlag           = 0;
+    pNote->taosNoteFd             = -1;
+    //pNote->taosNoteOpenInProgress = 0;
+
+    if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes, pNote) < 0)
         fprintf(stderr, "failed to init note file\n");
 
-    taosNotePrint("==================================================");
-    taosNotePrint("===================  new note  ===================");
-    taosNotePrint("==================================================");
+    taosNotePrint(pNote, "==================================================");
+    taosNotePrint(pNote, "===================  new note  ===================");
+    taosNotePrint(pNote, "==================================================");
 }
 
-void taosCloseNoteByFd(int oldFd);
-bool taosLockNote(int fd)
+void taosCloseNoteByFd(int oldFd, taosNoteInfo * pNote);
+bool taosLockNote(int fd, taosNoteInfo * pNote)
 {
     if (fd < 0) return false;
 
-    if (taosNoteFileNum > 1) {
+    if (pNote->taosNoteFileNum > 1) {
         int ret = (int)(flock(fd, LOCK_EX | LOCK_NB));
         if (ret == 0) {
             return true;
@@ -58,11 +66,11 @@ bool taosLockNote(int fd)
     return false;
 }
 
-void taosUnLockNote(int fd)
+void taosUnLockNote(int fd, taosNoteInfo * pNote)
 {
     if (fd < 0) return;
 
-    if (taosNoteFileNum > 1) {
+    if (pNote->taosNoteFileNum > 1) {
         flock(fd, LOCK_UN | LOCK_NB);
     }
 }
@@ -70,50 +78,51 @@ void taosUnLockNote(int fd)
 void *taosThreadToOpenNewNote(void *param)
 {
     char name[NOTE_FILE_NAME_LEN];
+    taosNoteInfo * pNote = (taosNoteInfo *)param;
 
-    taosNoteFlag ^= 1;
-    taosNoteLines = 0;
-    sprintf(name, "%s.%d", taosNoteName, taosNoteFlag);
+    pNote->taosNoteFlag ^= 1;
+    pNote->taosNoteLines = 0;
+    sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
 
     umask(0);
 
     int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
-    taosLockNote(fd);
+    taosLockNote(fd, pNote);
     lseek(fd, 0, SEEK_SET);
 
-    int oldFd = taosNoteFd;
-    taosNoteFd = fd;
-    taosNoteLines = 0;
-    taosNoteOpenInProgress = 0;
-    taosNotePrint("===============  new note is opened  =============");
+    int oldFd = pNote->taosNoteFd;
+    pNote->taosNoteFd = fd;
+    pNote->taosNoteLines = 0;
+    pNote->taosNoteOpenInProgress = 0;
+    taosNotePrint(pNote, "===============  new note is opened  =============");
 
-    taosCloseNoteByFd(oldFd);
+    taosCloseNoteByFd(oldFd, pNote);
     return NULL;
 }
 
-int taosOpenNewNote()
+int taosOpenNewNote(taosNoteInfo * pNote)
 {
-    pthread_mutex_lock(&taosNoteMutex);
+    pthread_mutex_lock(&pNote->taosNoteMutex);
 
-    if (taosNoteLines > taosNoteMaxLines && taosNoteOpenInProgress == 0) {
-        taosNoteOpenInProgress = 1;
+    if (pNote->taosNoteLines > pNote->taosNoteMaxLines && pNote->taosNoteOpenInProgress == 0) {
+        pNote->taosNoteOpenInProgress = 1;
 
-        taosNotePrint("===============  open new note  ==================");
+        taosNotePrint(pNote, "===============  open new note  ==================");
         pthread_t pattern;
         pthread_attr_t attr;
         pthread_attr_init(&attr);
         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-        pthread_create(&pattern, &attr, taosThreadToOpenNewNote, NULL);
+        pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void*)pNote);
         pthread_attr_destroy(&attr);
     }
 
-    pthread_mutex_unlock(&taosNoteMutex);
+    pthread_mutex_unlock(&pNote->taosNoteMutex);
 
-    return taosNoteFd;
+    return pNote->taosNoteFd;
 }
 
-bool taosCheckNoteIsOpen(char *noteName)
+bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote)
 {
     int exist = access(noteName, F_OK);
     if (exist != 0) {
@@ -126,8 +135,8 @@ bool taosCheckNoteIsOpen(char *noteName)
         return true;
     }
 
-    if (taosLockNote(fd)) {
-        taosUnLockNote(fd);
+    if (taosLockNote(fd, pNote)) {
+        taosUnLockNote(fd, pNote);
         close(fd);
         return false;
     }
@@ -137,80 +146,80 @@ bool taosCheckNoteIsOpen(char *noteName)
     }
 }
 
-void taosGetNoteName(char *fn)
+void taosGetNoteName(char *fn, taosNoteInfo * pNote)
 {
-    if (taosNoteFileNum > 1) {
-        for (int i = 0; i < taosNoteFileNum; i++) {
+    if (pNote->taosNoteFileNum > 1) {
+        for (int i = 0; i < pNote->taosNoteFileNum; i++) {
             char fileName[NOTE_FILE_NAME_LEN];
 
             sprintf(fileName, "%s%d.0", fn, i);
-            bool file1open = taosCheckNoteIsOpen(fileName);
+            bool file1open = taosCheckNoteIsOpen(fileName, pNote);
 
             sprintf(fileName, "%s%d.1", fn, i);
-            bool file2open = taosCheckNoteIsOpen(fileName);
+            bool file2open = taosCheckNoteIsOpen(fileName, pNote);
 
             if (!file1open && !file2open) {
-                sprintf(taosNoteName, "%s%d", fn, i);
+                sprintf(pNote->taosNoteName, "%s%d", fn, i);
                 return;
             }
         }
     }
 
-    strcpy(taosNoteName, fn);
+    strcpy(pNote->taosNoteName, fn);
 }
 
-int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum)
+int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote)
 {
     char name[NOTE_FILE_NAME_LEN] = "\0";
     struct stat  notestat0, notestat1;
     int size;
 
-    taosNoteMaxLines = maxLines;
-    taosNoteFileNum = maxNoteNum;
-    taosGetNoteName(fn);
+    pNote->taosNoteMaxLines = maxLines;
+    pNote->taosNoteFileNum = maxNoteNum;
+    taosGetNoteName(fn, pNote);
 
     strcpy(name, fn);
     strcat(name, ".0");
 
     // if none of the note files exist, open 0, if both exists, open the old one
     if (stat(name, &notestat0) < 0) {
-        taosNoteFlag = 0;
+        pNote->taosNoteFlag = 0;
     }
     else {
         strcpy(name, fn);
         strcat(name, ".1");
         if (stat(name, &notestat1) < 0) {
-            taosNoteFlag = 1;
+            pNote->taosNoteFlag = 1;
         }
         else {
-            taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
+            pNote->taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1;
         }
     }
 
-    sprintf(name, "%s.%d", taosNoteName, taosNoteFlag);
-    pthread_mutex_init(&taosNoteMutex, NULL);
+    sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag);
+    pthread_mutex_init(&pNote->taosNoteMutex, NULL);
 
     umask(0);
-    taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+    pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
-    if (taosNoteFd < 0) {
+    if (pNote->taosNoteFd < 0) {
         fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno));
         return -1;
     }
-    taosLockNote(taosNoteFd);
+    taosLockNote(pNote->taosNoteFd, pNote);
 
     // only an estimate for number of lines
     struct stat filestat;
-    fstat(taosNoteFd, &filestat);
+    fstat(pNote->taosNoteFd, &filestat);
     size = (int)filestat.st_size;
-    taosNoteLines = size / 60;
+    pNote->taosNoteLines = size / 60;
 
-    lseek(taosNoteFd, 0, SEEK_END);
+    lseek(pNote->taosNoteFd, 0, SEEK_END);
 
     return 0;
 }
 
-void taosNotePrint(const char * const format, ...)
+void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
 {
     va_list argpointer;
     char    buffer[MAX_NOTE_LINE_SIZE];
@@ -222,7 +231,7 @@ void taosNotePrint(const char * const format, ...)
     gettimeofday(&timeSecs, NULL);
     curTime = timeSecs.tv_sec;
     ptm = localtime_r(&curTime, &Tm);
-    len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec);
+    len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %lx ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec, pthread_self());
 
     va_start(argpointer, format);
     len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer);
@@ -233,26 +242,26 @@ void taosNotePrint(const char * const format, ...)
     buffer[len++] = '\n';
     buffer[len] = 0;
 
-    if (taosNoteFd >= 0)  {
-        twrite(taosNoteFd, buffer, (unsigned int)len);
+    if (pNote->taosNoteFd >= 0)  {
+        twrite(pNote->taosNoteFd, buffer, (unsigned int)len);
 
-        if (taosNoteMaxLines > 0) {
-            taosNoteLines++;
-            if ((taosNoteLines > taosNoteMaxLines) && (taosNoteOpenInProgress == 0))
-                taosOpenNewNote();
+        if (pNote->taosNoteMaxLines > 0) {
+            pNote->taosNoteLines++;
+            if ((pNote->taosNoteLines > pNote->taosNoteMaxLines) && (pNote->taosNoteOpenInProgress == 0))
+                taosOpenNewNote(pNote);
         }
     }
 }
 
-void taosCloseNote()
+void taosCloseNote(taosNoteInfo * pNote)
 {
-    taosCloseNoteByFd(taosNoteFd);
+    taosCloseNoteByFd(pNote->taosNoteFd, pNote);
 }
 
-void taosCloseNoteByFd(int fd)
+void taosCloseNoteByFd(int fd, taosNoteInfo * pNote)
 {
     if (fd >= 0) {
-        taosUnLockNote(fd);
+        taosUnLockNote(fd, pNote);
         close(fd);
     }
 }