From 03abae0c6d936414a6f18e336ec13c6eb9159fae Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 21 Nov 2019 17:18:14 +0800 Subject: [PATCH] [TBASE-1123] --- src/client/src/tscAsync.c | 5 +- src/client/src/tscSql.c | 3 + src/client/src/tscSystem.c | 10 +- src/inc/tglobalcfg.h | 2 + src/inc/tnote.h | 64 +++++++++++ src/modules/http/src/httpSql.c | 6 +- src/modules/http/src/httpSystem.c | 4 +- src/util/src/tglobalcfg.c | 5 + src/util/src/tnote.c | 177 ++++++++++++++++-------------- 9 files changed, 185 insertions(+), 91 deletions(-) create mode 100644 src/inc/tnote.h diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index aad0295959..8b324afc5e 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -24,6 +24,7 @@ #include "tsocket.h" #include "tsql.h" #include "tutil.h" +#include "tnote.h" void tscProcessFetchRow(SSchedMsg *pMsg); void tscProcessAsyncRetrieve(void *param, TAOS_RES *tres, int numOfRows); @@ -53,7 +54,9 @@ void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, tscError("sql string too long"); tscQueueAsyncError(fp, param); return; - } + } + + taosNotePrintTsc(sqlstr); SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj)); if (pSql == NULL) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 13c5669fec..be7c19ca98 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -27,6 +27,7 @@ #include "tsql.h" #include "ttimer.h" #include "tutil.h" +#include "tnote.h" TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { @@ -274,6 +275,8 @@ int taos_query(TAOS *taos, const char *sqlstr) { return pRes->code; } + taosNotePrintTsc(sqlstr); + void *sql = realloc(pSql->sqlstr, sqlLen + 1); if (sql == NULL) { pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 2c5ad2939a..60b90ac328 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -45,6 +45,10 @@ extern int tscEmbedded; int tscNumOfThreads; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +extern int tsTscEnableRecordSql; +extern int tsNumOfLogLines; +void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); + void tscCheckDiskUsage(void *para, void *unused) { taosGetDisk(); taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); @@ -85,6 +89,10 @@ void taos_init_imp() { taosSetCoreDump(); + if (tsTscEnableRecordSql != 0) { + taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); + } + #ifdef CLUSTER tscMgmtIpList.numOfIps = 2; strcpy(tscMgmtIpList.ipstr[0], tsMasterIp); @@ -354,4 +362,4 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { atomic_store_32(&lock, 0); return ret; -} \ No newline at end of file +} diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 577f3e8c89..8f0cf79fe6 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -150,6 +150,8 @@ extern int tsHttpEnableRecordSql; extern int tsTelegrafUseFieldNum; extern int tsAdminRowLimit; +extern int tsTscEnableRecordSql; + extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int tsMonitorInterval; diff --git a/src/inc/tnote.h b/src/inc/tnote.h new file mode 100644 index 0000000000..1b53be22a2 --- /dev/null +++ b/src/inc/tnote.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TNOTE_H +#define TDENGINE_TNOTE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "unistd.h" +#include "os.h" +#include "tutil.h" +#include "tglobalcfg.h" + +#define MAX_NOTE_LINE_SIZE 66000 +#define NOTE_FILE_NAME_LEN 300 + +typedef struct _taosNoteInfo { + int taosNoteFileNum ; + int taosNoteMaxLines; + int taosNoteLines; + char taosNoteName[NOTE_FILE_NAME_LEN]; + int taosNoteFlag; + int taosNoteFd; + int taosNoteOpenInProgress; + pthread_mutex_t taosNoteMutex; +}taosNoteInfo; + +void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...); + +extern taosNoteInfo m_HttpNote; +extern taosNoteInfo m_TscNote; + +extern int tsHttpEnableRecordSql; +extern int tsTscEnableRecordSql; + +#define taosNotePrintHttp(...) \ + if (tsHttpEnableRecordSql) { \ + taosNotePrint(&m_HttpNote, __VA_ARGS__); \ + } + +#define taosNotePrintTsc(...) \ + if (tsTscEnableRecordSql) { \ + taosNotePrint(&m_TscNote, __VA_ARGS__); \ + } + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/modules/http/src/httpSql.c b/src/modules/http/src/httpSql.c index 0e3b211abb..732d0179ff 100644 --- a/src/modules/http/src/httpSql.c +++ b/src/modules/http/src/httpSql.c @@ -24,11 +24,11 @@ #include "httpResp.h" #include "taos.h" #include "tsclient.h" +#include "tnote.h" void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos); void httpProcessMultiSql(HttpContext *pContext); -void taosNotePrint(const char * const format, ...); void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { HttpContext *pContext = (HttpContext *)param; @@ -165,7 +165,7 @@ void httpProcessMultiSql(HttpContext *pContext) { char *sql = httpGetCmdsString(pContext, cmd->sql); httpDump("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, sql); - taosNotePrint(sql); + taosNotePrintHttp(sql); taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext); } @@ -298,7 +298,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { httpDump("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, sql); - taosNotePrint(sql); + taosNotePrintHttp(sql); taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext); } diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c index df49251f13..c33cd44444 100644 --- a/src/modules/http/src/httpSystem.c +++ b/src/modules/http/src/httpSystem.c @@ -42,7 +42,7 @@ #endif static HttpServer *httpServer = NULL; -void taosInitNote(int numOfNoteLines, int maxNotes); +void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); int httpInitSystem() { taos_init(); @@ -61,7 +61,7 @@ int httpInitSystem() { pthread_mutex_init(&httpServer->serverMutex, NULL); if (tsHttpEnableRecordSql != 0) { - taosInitNote(tsNumOfLogLines / 10, 1); + taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note"); } restInitHandle(httpServer); adminInitHandle(httpServer); diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 99c2b8e530..286983b0b2 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -162,6 +162,8 @@ int tsHttpEnableRecordSql = 0; int tsTelegrafUseFieldNum = 0; int tsAdminRowLimit = 10240; +int tsTscEnableRecordSql = 0; + int tsRpcTimer = 300; int tsRpcMaxTime = 600; // seconds; @@ -767,6 +769,9 @@ static void doInitGlobalConfig() { 0, 255, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "tscEnableRecordSql", &tsTscEnableRecordSql, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 100000, 0, TSDB_CFG_UTYPE_NONE); // version info tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 709ac2742e..d12cc6e613 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -13,42 +13,50 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tutil.h" -#include "tglobalcfg.h" - -#define MAX_NOTE_LINE_SIZE 66000 -#define NOTE_FILE_NAME_LEN 300 - -static int taosNoteFileNum = 1; -static int taosNoteMaxLines = 0; -static int taosNoteLines = 0; -static char taosNoteName[NOTE_FILE_NAME_LEN]; -static int taosNoteFlag = 0; -static int taosNoteFd = -1; -static int taosNoteOpenInProgress = 0; -static pthread_mutex_t taosNoteMutex; -void taosNotePrint(const char * const format, ...); -int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum); - -void taosInitNote(int numOfNoteLines, int maxNotes) +#include "tnote.h" + +taosNoteInfo m_HttpNote; +taosNoteInfo m_TscNote; + +int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote); + +void taosInitNote(int numOfNoteLines, int maxNotes, char* lable) { + taosNoteInfo * pNote = NULL; char temp[128] = { 0 }; - sprintf(temp, "%s/taosnote", logDir); - if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes) < 0) + + if (strcasecmp(lable, "http_note") == 0) { + pNote = &m_HttpNote; + sprintf(temp, "%s/httpnote", logDir); + } else if (strcasecmp(lable, "tsc_note") == 0) { + pNote = &m_TscNote; + sprintf(temp, "%s/tscnote-%d", logDir, getpid()); + } else { + return; + } + + memset(pNote, 0, sizeof(taosNoteInfo)); + pNote->taosNoteFileNum = 1; + //pNote->taosNoteMaxLines = 0; + //pNote->taosNoteLines = 0; + //pNote->taosNoteFlag = 0; + pNote->taosNoteFd = -1; + //pNote->taosNoteOpenInProgress = 0; + + if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes, pNote) < 0) fprintf(stderr, "failed to init note file\n"); - taosNotePrint("=================================================="); - taosNotePrint("=================== new note ==================="); - taosNotePrint("=================================================="); + taosNotePrint(pNote, "=================================================="); + taosNotePrint(pNote, "=================== new note ==================="); + taosNotePrint(pNote, "=================================================="); } -void taosCloseNoteByFd(int oldFd); -bool taosLockNote(int fd) +void taosCloseNoteByFd(int oldFd, taosNoteInfo * pNote); +bool taosLockNote(int fd, taosNoteInfo * pNote) { if (fd < 0) return false; - if (taosNoteFileNum > 1) { + if (pNote->taosNoteFileNum > 1) { int ret = (int)(flock(fd, LOCK_EX | LOCK_NB)); if (ret == 0) { return true; @@ -58,11 +66,11 @@ bool taosLockNote(int fd) return false; } -void taosUnLockNote(int fd) +void taosUnLockNote(int fd, taosNoteInfo * pNote) { if (fd < 0) return; - if (taosNoteFileNum > 1) { + if (pNote->taosNoteFileNum > 1) { flock(fd, LOCK_UN | LOCK_NB); } } @@ -70,50 +78,51 @@ void taosUnLockNote(int fd) void *taosThreadToOpenNewNote(void *param) { char name[NOTE_FILE_NAME_LEN]; + taosNoteInfo * pNote = (taosNoteInfo *)param; - taosNoteFlag ^= 1; - taosNoteLines = 0; - sprintf(name, "%s.%d", taosNoteName, taosNoteFlag); + pNote->taosNoteFlag ^= 1; + pNote->taosNoteLines = 0; + sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); umask(0); int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - taosLockNote(fd); + taosLockNote(fd, pNote); lseek(fd, 0, SEEK_SET); - int oldFd = taosNoteFd; - taosNoteFd = fd; - taosNoteLines = 0; - taosNoteOpenInProgress = 0; - taosNotePrint("=============== new note is opened ============="); + int oldFd = pNote->taosNoteFd; + pNote->taosNoteFd = fd; + pNote->taosNoteLines = 0; + pNote->taosNoteOpenInProgress = 0; + taosNotePrint(pNote, "=============== new note is opened ============="); - taosCloseNoteByFd(oldFd); + taosCloseNoteByFd(oldFd, pNote); return NULL; } -int taosOpenNewNote() +int taosOpenNewNote(taosNoteInfo * pNote) { - pthread_mutex_lock(&taosNoteMutex); + pthread_mutex_lock(&pNote->taosNoteMutex); - if (taosNoteLines > taosNoteMaxLines && taosNoteOpenInProgress == 0) { - taosNoteOpenInProgress = 1; + if (pNote->taosNoteLines > pNote->taosNoteMaxLines && pNote->taosNoteOpenInProgress == 0) { + pNote->taosNoteOpenInProgress = 1; - taosNotePrint("=============== open new note =================="); + taosNotePrint(pNote, "=============== open new note =================="); pthread_t pattern; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, NULL); + pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void*)pNote); pthread_attr_destroy(&attr); } - pthread_mutex_unlock(&taosNoteMutex); + pthread_mutex_unlock(&pNote->taosNoteMutex); - return taosNoteFd; + return pNote->taosNoteFd; } -bool taosCheckNoteIsOpen(char *noteName) +bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote) { int exist = access(noteName, F_OK); if (exist != 0) { @@ -126,8 +135,8 @@ bool taosCheckNoteIsOpen(char *noteName) return true; } - if (taosLockNote(fd)) { - taosUnLockNote(fd); + if (taosLockNote(fd, pNote)) { + taosUnLockNote(fd, pNote); close(fd); return false; } @@ -137,80 +146,80 @@ bool taosCheckNoteIsOpen(char *noteName) } } -void taosGetNoteName(char *fn) +void taosGetNoteName(char *fn, taosNoteInfo * pNote) { - if (taosNoteFileNum > 1) { - for (int i = 0; i < taosNoteFileNum; i++) { + if (pNote->taosNoteFileNum > 1) { + for (int i = 0; i < pNote->taosNoteFileNum; i++) { char fileName[NOTE_FILE_NAME_LEN]; sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName); + bool file1open = taosCheckNoteIsOpen(fileName, pNote); sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName); + bool file2open = taosCheckNoteIsOpen(fileName, pNote); if (!file1open && !file2open) { - sprintf(taosNoteName, "%s%d", fn, i); + sprintf(pNote->taosNoteName, "%s%d", fn, i); return; } } } - strcpy(taosNoteName, fn); + strcpy(pNote->taosNoteName, fn); } -int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum) +int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote) { char name[NOTE_FILE_NAME_LEN] = "\0"; struct stat notestat0, notestat1; int size; - taosNoteMaxLines = maxLines; - taosNoteFileNum = maxNoteNum; - taosGetNoteName(fn); + pNote->taosNoteMaxLines = maxLines; + pNote->taosNoteFileNum = maxNoteNum; + taosGetNoteName(fn, pNote); strcpy(name, fn); strcat(name, ".0"); // if none of the note files exist, open 0, if both exists, open the old one if (stat(name, ¬estat0) < 0) { - taosNoteFlag = 0; + pNote->taosNoteFlag = 0; } else { strcpy(name, fn); strcat(name, ".1"); if (stat(name, ¬estat1) < 0) { - taosNoteFlag = 1; + pNote->taosNoteFlag = 1; } else { - taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1; + pNote->taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1; } } - sprintf(name, "%s.%d", taosNoteName, taosNoteFlag); - pthread_mutex_init(&taosNoteMutex, NULL); + sprintf(name, "%s.%d", pNote->taosNoteName, pNote->taosNoteFlag); + pthread_mutex_init(&pNote->taosNoteMutex, NULL); umask(0); - taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pNote->taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (taosNoteFd < 0) { + if (pNote->taosNoteFd < 0) { fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno)); return -1; } - taosLockNote(taosNoteFd); + taosLockNote(pNote->taosNoteFd, pNote); // only an estimate for number of lines struct stat filestat; - fstat(taosNoteFd, &filestat); + fstat(pNote->taosNoteFd, &filestat); size = (int)filestat.st_size; - taosNoteLines = size / 60; + pNote->taosNoteLines = size / 60; - lseek(taosNoteFd, 0, SEEK_END); + lseek(pNote->taosNoteFd, 0, SEEK_END); return 0; } -void taosNotePrint(const char * const format, ...) +void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...) { va_list argpointer; char buffer[MAX_NOTE_LINE_SIZE]; @@ -222,7 +231,7 @@ void taosNotePrint(const char * const format, ...) gettimeofday(&timeSecs, NULL); curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec); + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %lx ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec, pthread_self()); va_start(argpointer, format); len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); @@ -233,26 +242,26 @@ void taosNotePrint(const char * const format, ...) buffer[len++] = '\n'; buffer[len] = 0; - if (taosNoteFd >= 0) { - twrite(taosNoteFd, buffer, (unsigned int)len); + if (pNote->taosNoteFd >= 0) { + twrite(pNote->taosNoteFd, buffer, (unsigned int)len); - if (taosNoteMaxLines > 0) { - taosNoteLines++; - if ((taosNoteLines > taosNoteMaxLines) && (taosNoteOpenInProgress == 0)) - taosOpenNewNote(); + if (pNote->taosNoteMaxLines > 0) { + pNote->taosNoteLines++; + if ((pNote->taosNoteLines > pNote->taosNoteMaxLines) && (pNote->taosNoteOpenInProgress == 0)) + taosOpenNewNote(pNote); } } } -void taosCloseNote() +void taosCloseNote(taosNoteInfo * pNote) { - taosCloseNoteByFd(taosNoteFd); + taosCloseNoteByFd(pNote->taosNoteFd, pNote); } -void taosCloseNoteByFd(int fd) +void taosCloseNoteByFd(int fd, taosNoteInfo * pNote) { if (fd >= 0) { - taosUnLockNote(fd); + taosUnLockNote(fd, pNote); close(fd); } } -- GitLab