Merge pull request #7692 from taosdata/feature/TD-6285-master

[TD-6285]<feature>: added db_name for url while restful was used

src/client/CMakeLists.txt

@@ -4,6 +4,8 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 INCLUDE_DIRECTORIES(jni)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 IF (TD_LINUX)

src/client/inc/tscUtil.h

@@ -361,6 +361,8 @@ SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
 
 void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id);
 
+char* cloneCurrentDBName(SSqlObj* pSql);
+
 #ifdef __cplusplus
 }
 #endif

src/client/inc/tsclient.h

@@ -38,6 +38,11 @@ extern "C" {
 #include "qUtil.h"
 #include "tcmdtype.h"
 
+typedef enum {
+  TAOS_REQ_FROM_SHELL,
+  TAOS_REQ_FROM_HTTP
+} SReqOrigin;
+
 // forward declaration
 struct SSqlInfo;
 
@@ -341,6 +346,7 @@ typedef struct STscObj {
   SRpcCorEpSet      *tscCorMgmtEpSet;
   pthread_mutex_t    mutex;
   int32_t            numOfObj; // number of sqlObj from this tscObj
+  SReqOrigin         from;
 } STscObj;
 
 typedef struct SSubqueryState {

src/client/src/tscSQLParser.c

@@ -70,7 +70,6 @@ static int  convertTimestampStrToInt64(tVariant *pVar, int32_t precision);
 static bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, uint8_t precision);
 
 static bool has(SArray* pFieldList, int32_t startIdx, const char* name);
-static char* cloneCurrentDBName(SSqlObj* pSql);
 static int32_t getDelimiterIndex(SStrToken* pTableName);
 static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd);
 static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd);
@@ -1675,14 +1674,6 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) {
 
 static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; }
 
-static char* cloneCurrentDBName(SSqlObj* pSql) {
-  pthread_mutex_lock(&pSql->pTscObj->mutex);
-  char *p = strdup(pSql->pTscObj->db);  
-  pthread_mutex_unlock(&pSql->pTscObj->mutex);
-
-  return p;
-}
-
 /* length limitation, strstr cannot be applied */
 static int32_t getDelimiterIndex(SStrToken* pTableName) {
   for (uint32_t i = 0; i < pTableName->n; ++i) {

src/client/src/tscServer.c

@@ -1532,7 +1532,6 @@ int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
 }
 
 int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
-  STscObj *pObj = pSql->pTscObj;
   SSqlCmd *pCmd = &pSql->cmd;
   pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW;
   pCmd->payloadLen = sizeof(SShowMsg) + 100;
@@ -1555,9 +1554,9 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   if (tNameIsEmpty(&pTableMetaInfo->name)) {
-    pthread_mutex_lock(&pObj->mutex);
-    tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));  
-    pthread_mutex_unlock(&pObj->mutex);
+    char *p = cloneCurrentDBName(pSql);
+    tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db));
+    tfree(p);
   } else {
     tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db);
   }

src/client/src/tscUtil.c

@@ -29,6 +29,7 @@
 #include "tsclient.h"
 #include "ttimer.h"
 #include "ttokendef.h"
+#include "httpInt.h"
 
 static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
 
@@ -5040,3 +5041,31 @@ void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id) {
   taosHashRemove(tscTableMetaMap, fname, len);
   tscDebug("0x%"PRIx64" remove table meta %s, numOfRemain:%d", id, fname, (int32_t) taosHashGetSize(tscTableMetaMap));
 }
+
+char* cloneCurrentDBName(SSqlObj* pSql) {
+  char        *p = NULL;
+  HttpContext *pCtx = NULL;
+
+  pthread_mutex_lock(&pSql->pTscObj->mutex);
+  STscObj *pTscObj = pSql->pTscObj;
+  switch (pTscObj->from) {
+  case TAOS_REQ_FROM_HTTP:
+    pCtx = pSql->param;
+    if (pCtx && pCtx->db[0] != '\0') {
+      char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] = {0};
+      int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db);
+      assert(len <= sizeof(db));
+
+      p = strdup(db);
+    }
+    break;
+  default:
+    break;
+  }
+  if (p == NULL) {
+    p = strdup(pSql->pTscObj->db);
+  }
+  pthread_mutex_unlock(&pSql->pTscObj->mutex);
+
+  return p;
+}

src/common/inc/tglobal.h

@@ -125,6 +125,7 @@ extern int32_t  tsHttpMaxThreads;
 extern int8_t   tsHttpEnableCompress;
 extern int8_t   tsHttpEnableRecordSql;
 extern int8_t   tsTelegrafUseFieldNum;
+extern int8_t   tsHttpDbNameMandatory;
 
 // mqtt
 extern int8_t tsEnableMqttModule;

src/common/src/tglobal.c

@@ -165,6 +165,7 @@ int32_t  tsHttpMaxThreads = 2;
 int8_t   tsHttpEnableCompress = 1;
 int8_t   tsHttpEnableRecordSql = 0;
 int8_t   tsTelegrafUseFieldNum = 0;
+int8_t   tsHttpDbNameMandatory = 0;
 
 // mqtt
 int8_t tsEnableMqttModule = 0;  // not finished yet, not started it by default
@@ -1267,6 +1268,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "httpDbNameMandatory";
+  cfg.ptr = &tsHttpDbNameMandatory;
+  cfg.valType = TAOS_CFG_VTYPE_INT8;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // debug flag
   cfg.option = "numOfLogLines";
   cfg.ptr = &tsNumOfLogLines;

src/plugins/http/CMakeLists.txt

@@ -6,6 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 

src/plugins/http/inc/httpInt.h

@@ -150,6 +150,7 @@ typedef struct HttpContext {
   char         ipstr[22];
   char         user[TSDB_USER_LEN];  // parsed from auth token or login message
   char         pass[HTTP_PASSWORD_LEN];
+  char         db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN];
   TAOS *       taos;
   void *       ppContext;
   HttpSession *session;

src/plugins/http/inc/httpRestHandle.h

@@ -22,12 +22,12 @@
 #include "httpResp.h"
 #include "httpSql.h"
 
-#define REST_ROOT_URL_POS   0
-#define REST_ACTION_URL_POS 1
-#define REST_USER_URL_POS   2
-#define REST_PASS_URL_POS   3
+#define REST_ROOT_URL_POS         0
+#define REST_ACTION_URL_POS       1
+#define REST_USER_USEDB_URL_POS   2
+#define REST_PASS_URL_POS         3
 
 void restInitHandle(HttpServer* pServer);
 bool restProcessRequest(struct HttpContext* pContext);
 
-#endif
\ No newline at end of file
+#endif

src/plugins/http/src/httpRestHandle.c

@@ -19,6 +19,7 @@
 #include "httpLog.h"
 #include "httpRestHandle.h"
 #include "httpRestJson.h"
+#include "tglobal.h"
 
 static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest};
 static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest};
@@ -62,11 +63,11 @@ void restInitHandle(HttpServer* pServer) {
 
 bool restGetUserFromUrl(HttpContext* pContext) {
   HttpParser* pParser = pContext->parser;
-  if (pParser->path[REST_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].pos <= 0) {
+  if (pParser->path[REST_USER_USEDB_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_USEDB_URL_POS].pos <= 0) {
     return false;
   }
 
-  tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].str, TSDB_USER_LEN);
+  tstrncpy(pContext->user, pParser->path[REST_USER_USEDB_URL_POS].str, TSDB_USER_LEN);
   return true;
 }
 
@@ -107,6 +108,24 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
   HttpSqlCmd* cmd = &(pContext->singleCmd);
   cmd->nativSql = sql;
 
+  /* find if there is db_name in url */
+  pContext->db[0] = '\0';
+
+  HttpString *path = &pContext->parser->path[REST_USER_USEDB_URL_POS];
+  if (tsHttpDbNameMandatory) {
+    if (path->pos == 0) {
+      httpError("context:%p, fd:%d, user:%s, database name is mandatory", pContext, pContext->fd, pContext->user);
+      httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL);
+      return false;
+    }
+  }
+
+  if (path->pos > 0 && !(strlen(sql) > 4 && (sql[0] == 'u' || sql[0] == 'U') &&
+      (sql[1] == 's' || sql[1] == 'S') && (sql[2] == 'e' || sql[2] == 'E') && sql[3] == ' '))
+  {
+    snprintf(pContext->db, /*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN, "%s", path->str);
+  }
+
   pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
   if (timestampFmt == REST_TIMESTAMP_FMT_LOCAL_STRING) {
     pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;

src/plugins/http/src/httpSql.c

@@ -419,6 +419,11 @@ void httpProcessRequest(HttpContext *pContext) {
                    &(pContext->taos));
     httpDebug("context:%p, fd:%d, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, pContext->user,
               pContext->taos);
+
+    if (pContext->taos != NULL) {
+      STscObj *pObj = pContext->taos;
+      pObj->from = TAOS_REQ_FROM_HTTP;
+    }
   } else {
     httpExecCmd(pContext);
   }

tests/pytest/fulltest.sh

@@ -169,6 +169,10 @@ python3 test.py -f tools/taosdemoTestSampleData.py
 python3 test.py -f tools/taosdemoTestInterlace.py
 python3 test.py -f tools/taosdemoTestQuery.py
 
+# restful test for python
+python3 test.py -f restful/restful_bind_db1.py
+python3 test.py -f restful/restful_bind_db2.py
+
 # nano support
 python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py
 python3 test.py -f tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoQuery.py

tests/pytest/restful/restful_bind_db1.py

+# #################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+
+# #################################################################
+
+# -*- coding: utf-8 -*-
+
+# TODO: after TD-4518 and TD-4510 is resolved, add the exception test case for these situations
+
+from distutils.log import error
+import sys
+
+from requests.api import head
+from requests.models import Response
+from util.log import *
+from util.cases import *
+from util.sql import *
+import time, datetime
+import requests, json
+import threading
+import string
+import random
+ 
+
+def check_unbind_db(url, data, header):
+    resp = requests.post(url, data, headers = header )
+    resp.encoding='utf-8'
+    resp = eval(resp.text)
+    status = resp['status']
+    desc = resp['desc']
+    sqls = data
+    if status=="error" and desc == "invalid url format":
+        print(" %s : check pass" %sqls)
+    else:
+        printf(" error occured , ")
+        sys.exit()
+
+def check_bind_db(url, data, header):
+    resp = requests.post(url, data, headers = header )
+    resp.encoding='utf-8'
+    resp_dict = eval(resp.text)
+    status = resp_dict['status']
+    if status =="succ":
+        print("%s  run success!"%data)
+        # print(resp.text)
+    else :
+        print("%s run failed !"%data)
+        print(resp.text)
+        sys.exit()
+        
+class TDTestCase():
+    updatecfgDict={'httpDbNameMandatory':1}  
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+    def run(self):
+        tdSql.prepare()
+        tdSql.execute('reset query cache')
+        tdSql.execute('drop database if exists test')
+        tdSql.execute('drop database if exists db')
+        tdSql.execute('drop database if exists des')
+        tdSql.execute('create database test')
+        tdSql.execute('create database des')
+
+        header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
+        url = "http://127.0.0.1:6041/rest/sql/"
+
+        # test with no bind databases
+
+        sqls = ["show databases;",
+        "use test", 
+        "show tables;",
+        "show dnodes;",
+        "show vgroups;",
+        "create database db;",
+        "drop database db;",
+        "select client_version();" ,
+        "use test",
+        "ALTER DATABASE test COMP 2;",
+        "create table tb (ts timestamp, id int , data double)",
+        "insert into tb values (now , 1, 1.0) ",
+        "select * from tb",
+        "show test.tables",
+        "show tables",
+        "insert into tb values (now , 2, 2.0) ",
+        "create table test.tb (ts timestamp, id int , data double)",
+        "insert into test.tb values (now , 2, 2.0) ",
+        "select * from tb",
+        "select * from test.tb"]
+        
+        for sql in sqls:
+            print("===================")
+            check_unbind_db(url,sql,header)
+
+        print("==================="*5)
+        print("                        check bind db about restful                  ")
+        print("==================="*5)
+        url = "http://127.0.0.1:6041/rest/sql/des"
+        for sql in sqls:
+            print("===================")
+            check_bind_db(url,sql,header)
+        # check data
+        tdSql.query("select * from test.tb")
+        tdSql.checkRows(1)
+        tdSql.query("select * from des.tb")
+        tdSql.checkRows(2)
+
+        os.system('sudo timedatectl set-ntp on')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/restful/restful_bind_db2.py

+# #################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+
+# #################################################################
+
+# -*- coding: utf-8 -*-
+
+# TODO: after TD-4518 and TD-4510 is resolved, add the exception test case for these situations
+
+from distutils.log import error
+import sys
+
+from requests.api import head
+from requests.models import Response
+from util.log import *
+from util.cases import *
+from util.sql import *
+import time, datetime
+import requests, json
+import threading
+import string
+import random
+
+def check_res(url, data, header):
+    resp = requests.post(url, data, headers = header )
+    resp.encoding='utf-8'
+    resp_dict = eval(resp.text)
+    status = resp_dict['status']
+    if status =="succ":
+        print("%s  run success!"%data)
+        # print(resp.text)
+    else :
+        print("%s run failed !"%data)
+        print(resp.text)
+        sys.exit()
+        
+class TDTestCase():
+    # updatecfgDict={'httpDbNameMandatory':0}  
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        tdSql.execute('reset query cache')
+        tdSql.execute('drop database if exists test')
+        tdSql.execute('drop database if exists db')
+        tdSql.execute('drop database if exists des')
+        tdSql.execute('create database test')
+        tdSql.execute('create database des')
+        header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
+        url = "http://127.0.0.1:6041/rest/sql/"
+
+        # test with no bind databases
+
+        sqls = ["show databases;",
+        "use test", 
+        "show tables;",
+        "show dnodes;",
+        "show vgroups;",
+        "create database db;",
+        "drop database db;",
+        "select client_version();" ,
+        "use test",
+        "ALTER DATABASE test COMP 2;",
+        "create table tb (ts timestamp, id int , data double)",
+        "insert into tb values (now , 1, 1.0) ",
+        "select * from tb",
+        "show test.tables",
+        "show tables",
+        "insert into tb values (now , 2, 2.0) ",
+        "create table test.tb (ts timestamp, id int , data double)",
+        "insert into test.tb values (now , 3, 3.0) ",
+        "select * from tb",
+        "select * from test.tb",
+        "create table des.tb (ts timestamp, id int , data double)",
+        "insert into des.tb values (now , 3, 3.0)"]
+        for sql in sqls:
+            print("===================")
+            if sql == "create table test.tb (ts timestamp, id int , data double)":
+                resp = requests.post(url, sql, headers = header )
+                print(resp.text)
+                print ("%s run occur error as expect ,check pass!" %(sql))
+            else:
+                check_res(url,sql,header)
+
+        tdSql.query("select * from test.tb")
+        tdSql.checkRows(3)
+        tdSql.query("select * from des.tb")
+        tdSql.checkRows(1)
+
+        print("==================="*5)
+        print("                        check bind db about restful                  ")
+        print("==================="*5)
+
+        tdSql.execute('reset query cache')
+        tdSql.execute('drop database if exists test')
+        tdSql.execute('drop database if exists db')
+        tdSql.execute('drop database if exists des')
+        tdSql.execute('create database test')
+        tdSql.execute('create database des')
+        
+        url = "http://127.0.0.1:6041/rest/sql/des"
+        for sql in sqls:
+            print("===================")
+            if sql in ["create table des.tb (ts timestamp, id int , data double)"]:
+                resp = requests.post(url, sql, headers = header )
+                print(resp.text)
+                print ("%s run occur error as expect ,check pass!" %(sql))
+            else:
+                check_res(url,sql,header)
+        # check data
+        tdSql.query("select * from test.tb")
+        tdSql.checkRows(1)
+        tdSql.query("select * from des.tb")
+        tdSql.checkRows(3)
+
+        os.system('sudo timedatectl set-ntp on')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())