Merge pull request #9052 from taosdata/feature/3.0_wxy

TD-11819 mock catalog function for parser ut

source/libs/parser/inc/insertParser.h

@@ -16,8 +16,16 @@
 #ifndef TDENGINE_INSERTPARSER_H
 #define TDENGINE_INSERTPARSER_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "parser.h"
 
 int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // TDENGINE_INSERTPARSER_H

source/libs/parser/src/insertParser.c

@@ -145,16 +145,10 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b
 }
 
 static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) {
-  SInsertStmtInfo *info = calloc(1, sizeof(SQueryStmtInfo));
-  if (NULL == info) {
+  *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo));
+  if (NULL == *pInsertInfo) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
-  // info->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
-  // if (NULL == info->pTableBlockHashList) {
-  //   tfree(info);
-  //   return TSDB_CODE_TSC_OUT_OF_MEMORY;
-  // }
-  *pInsertInfo = info;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -808,7 +802,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     // no data in the sql string anymore.
     if (sToken.n == 0) {
       if (0 == pCxt->totalNum) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
+        return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");;
       }
       break;
     }

source/libs/parser/test/insertTest.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include "insertParser.h"
+#include "mockCatalog.h"
+
+using namespace std;
+using namespace testing;
+
+namespace {
+  string toString(int32_t code) {
+    return tstrerror(code);
+  }
+}
+
+// syntax:
+// INSERT INTO
+//   tb_name
+//       [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
+//       [(field1_name, ...)]
+//       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
+//   [...];
+class InsertTest : public Test {
+protected:
+  void bind(const char* sql) {
+    reset();
+    cxt.pSql = sql;
+    cxt.sqlLen = strlen(sql);
+  }
+
+  int32_t run() {
+    code = parseInsertSql(&cxt, &res);
+    if (code != TSDB_CODE_SUCCESS) {
+      cout << "code:" << toString(code) << ", msg:" << errMagBuf << endl;
+    }
+    return code;
+  }
+
+  SInsertStmtInfo* reslut() {
+    return res;
+  }
+
+private:
+  static const int max_err_len = 1024;
+
+  void reset() {
+    memset(&cxt, 0, sizeof(cxt));
+    memset(errMagBuf, 0, max_err_len);
+    cxt.pMsg = errMagBuf;
+    cxt.msgLen = max_err_len;
+    code = TSDB_CODE_SUCCESS;
+    res = nullptr;
+  }
+
+  char errMagBuf[max_err_len];
+  SParseContext cxt;
+  int32_t code;
+  SInsertStmtInfo* res;
+};
+
+// INSERT INTO tb_name VALUES (field1_value, ...)
+TEST_F(InsertTest, simpleTest) {
+  bind("insert into .. values (...)");
+  ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
+  SInsertStmtInfo* res = reslut();
+  // todo check
+}
+
+TEST_F(InsertTest, toleranceTest) {
+  bind("insert into");
+  ASSERT_NE(run(), TSDB_CODE_SUCCESS);
+  bind("insert into t");
+  ASSERT_NE(run(), TSDB_CODE_SUCCESS);
+}

source/libs/parser/test/mockCatalog.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mockCatalog.h"
+
+#include <iostream>
+
+void generateMetaData(MockCatalogService* mcs) {
+  {
+    ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, MockCatalogService::numOfDataTypes)
+        .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP);
+    for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) {
+      if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) {
+        continue;
+      }
+      builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type);
+    }
+    builder.done();
+  }
+  {
+    ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, MockCatalogService::numOfDataTypes, 2)
+        .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(2).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP);
+    for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) {
+      if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) {
+        continue;
+      }
+      builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type);
+    }
+    builder.done();
+  }
+  mcs->showTables();
+}
+
+struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
+  return mockCatalogService->getCatalogHandle(pMgmtEps);
+}
+
+int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+  return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData);
+}

source/libs/parser/test/mockCatalog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MOCK_CATALOG_H
+#define MOCK_CATALOG_H
+
+#include "mockCatalogService.h"
+
+void generateMetaData(MockCatalogService* mcs);
+
+// mock
+struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps);
+int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData);
+
+#endif  // MOCK_CATALOG_H

source/libs/parser/test/mockCatalogService.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mockCatalogService.h"
+
+#include <iomanip>
+#include <iostream>
+#include <map>
+
+#include "ttypes.h"
+
+std::unique_ptr<MockCatalogService> mockCatalogService;
+
+class TableBuilder : public ITableBuilder {
+public:
+  virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) {
+    assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns);
+    SSchema* col = meta_->schema + colIndex_;
+    col->type = type;
+    col->colId = colIndex_++;
+    col->bytes = bytes;
+    strcpy(col->name, name.c_str());
+    return *this;
+  }
+
+  virtual TableBuilder& setVgid(int16_t vgid) {
+    meta_->vgId = vgid;
+    return *this;
+  }
+
+  virtual TableBuilder& setPrecision(uint8_t precision) {
+    meta_->tableInfo.precision = precision;
+    return *this;
+  }
+
+  virtual void done() {
+    meta_->tableInfo.rowSize = rowsize_;
+  }
+
+private:
+  friend class MockCatalogServiceImpl;
+
+  static std::unique_ptr<TableBuilder> createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+    STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags));
+    if (nullptr == meta) {
+      throw std::bad_alloc();
+    }
+    meta->tableType = tableType;
+    meta->tableInfo.numOfTags = numOfTags;
+    meta->tableInfo.numOfColumns = numOfColumns;
+    return std::unique_ptr<TableBuilder>(new TableBuilder(meta));
+  }
+
+  TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) {
+  }
+
+  STableMeta* table() {
+    return meta_;
+  }
+
+  int32_t colIndex_;
+  int32_t rowsize_;
+  STableMeta* meta_;
+};
+
+class MockCatalogServiceImpl {
+public:
+  static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]);
+
+  MockCatalogServiceImpl() {
+  }
+
+  struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
+    return (struct SCatalog*)0x01;
+  }
+
+  int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+    return 0;
+  }
+
+  TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+    builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags);
+    meta_[db][tbname].reset(builder_->table());
+    meta_[db][tbname]->uid = id_++;
+    return *(builder_.get());
+  }
+
+  void showTables() const {
+    // number of forward fills
+    #define NOF(n) ((n) / 2)
+    // number of backward fills
+    #define NOB(n) ((n) % 2 ? (n) / 2 + 1 : (n) / 2)
+    // center aligned
+    #define CA(n, s) std::setw(NOF((n) - (s).length())) << "" << (s) << std::setw(NOB((n) - (s).length())) << "" << "|"
+    // string field length
+    #define SFL 20
+    // string field header
+    #define SH(h) CA(SFL, std::string(h))
+    // string field
+    #define SF(n) CA(SFL, n)
+    // integer field length
+    #define IFL 10
+    // integer field header
+    #define IH(i) CA(IFL, std::string(i))
+    // integer field
+    #define IF(i) CA(IFL, std::to_string(i))
+    // split line
+    #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ')
+
+    for (const auto& db : meta_) {
+      std::cout << SH("Database") << SH("Table") << SH("Type") << SH("Precision") << IH(std::string("Vgid")) << std::endl;
+      std::cout << SL(4, 1) << std::endl;
+      for (const auto& table : db.second) {
+        std::cout << SF(db.first) << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl;
+        // int16_t numOfFields = table.second->tableInfo.numOfTags + table.second->tableInfo.numOfColumns;
+        // for (int16_t i = 0; i < numOfFields; ++i) {
+        //   const SSchema* schema = table.second->schema + i;
+        //   std::cout << schema->name << " " << schema->type << " " << schema->bytes << std::endl;
+        // }
+      }
+    }
+  }
+
+private:
+  std::string ttToString(int8_t tableType) const {
+    switch (tableType) {
+      case TSDB_SUPER_TABLE:
+        return "super table";
+      case TSDB_CHILD_TABLE:
+        return "child table";
+      case TSDB_NORMAL_TABLE:
+        return "normal table";
+      default:
+        return "unknown";
+    }
+  }
+
+  std::string pToString(uint8_t precision) const {
+    switch (precision) {
+      case TSDB_TIME_PRECISION_MILLI:
+        return "millisecond";
+      case TSDB_TIME_PRECISION_MICRO:
+        return "microsecond";
+      case TSDB_TIME_PRECISION_NANO:
+        return "nanosecond";
+      default:
+        return "unknown";
+    }
+  }
+
+  uint64_t id_;
+  std::unique_ptr<TableBuilder> builder_;
+  std::map<std::string, std::map<std::string, std::shared_ptr<STableMeta> > > meta_;
+};
+
+MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {
+}
+
+MockCatalogService::~MockCatalogService() {
+}
+
+struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) {
+  return impl_->getCatalogHandle(pMgmtEps);
+}
+
+int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+  return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData);
+}
+
+ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+  return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags);
+}
+
+void MockCatalogService::showTables() const {
+  impl_->showTables();
+}
\ No newline at end of file

source/libs/parser/test/mockCatalogService.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MOCK_CATALOG_SERVICE_H
+#define MOCK_CATALOG_SERVICE_H
+
+#include <memory>
+#include <string>
+
+#include "catalog.h"
+
+class ITableBuilder {
+public:
+  ITableBuilder& addTag(const std::string& name, int8_t type) {
+    return addColumn(name, type, tDataTypes[type].bytes);
+  }
+
+  ITableBuilder& addTag(const std::string& name, int8_t type, int32_t bytes) {
+    return addColumn(name, type, bytes);
+  }
+
+  ITableBuilder& addColumn(const std::string& name, int8_t type) {
+    return addColumn(name, type, tDataTypes[type].bytes);
+  }
+
+  virtual ITableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) = 0;
+  virtual ITableBuilder& setVgid(int16_t vgid) = 0;
+  virtual ITableBuilder& setPrecision(uint8_t precision) = 0;
+  virtual void done() = 0;
+};
+
+class MockCatalogServiceImpl;
+
+class MockCatalogService {
+public:
+  static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]);
+
+  MockCatalogService();
+  ~MockCatalogService();
+  struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps);
+  int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData);
+  ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0);
+  void showTables() const;
+
+private:
+  std::unique_ptr<MockCatalogServiceImpl> impl_;
+};
+
+extern std::unique_ptr<MockCatalogService> mockCatalogService;
+
+#endif  // MOCK_CATALOG_SERVICE_H

source/libs/parser/test/parserMain.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "mockCatalog.h"
+
+class ParserEnv : public testing::Environment {
+public:
+  virtual void SetUp() {
+    mockCatalogService.reset(new MockCatalogService());
+    generateMetaData(mockCatalogService.get());
+  }
+
+  virtual void TearDown() {
+    mockCatalogService.reset();
+  }
+
+  ParserEnv() {}
+  virtual ~ParserEnv() {}
+};
+
+int main(int argc, char* argv[]) {
+	testing::AddGlobalTestEnvironment(new ParserEnv());
+	testing::InitGoogleTest(&argc, argv);
+	return RUN_ALL_TESTS();
+}

source/libs/parser/test/tokenizerTest.cpp

@@ -79,11 +79,6 @@ static void _init_tvariant_nchar(SVariant* t) {
   t->nLen = twcslen(t->wpz);
 }
 
-int main(int argc, char** argv) {
-  testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
 TEST(testCase, validateToken_test) {
   char t01[] = "abc";
   EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);