From 0ed08405c8601b9d8a17fef5300629e44e0e7fe8 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 10 Mar 2020 14:39:11 +0800 Subject: [PATCH] refactor code --- src/CMakeLists.txt | 2 +- src/common/inc/dataformat.h | 42 ++++++++++- src/common/src/dataformat.c | 130 ++++++++++++++++++++++++++++++++ src/common/src/ttypes.c | 15 +++- src/inc/taosdef.h | 2 + src/vnode/common/inc/schema.h | 68 ----------------- src/vnode/common/inc/type.h | 37 --------- src/vnode/common/src/schema.c | 136 ---------------------------------- src/vnode/tsdb/inc/tsdb.h | 4 +- src/vnode/tsdb/inc/tsdbMeta.h | 4 +- src/vnode/tsdb/src/tsdbMain.c | 5 +- 11 files changed, 194 insertions(+), 251 deletions(-) delete mode 100644 src/vnode/common/inc/schema.h delete mode 100644 src/vnode/common/inc/type.h delete mode 100644 src/vnode/common/src/schema.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7eba0cd736..606ef040d8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,6 +14,6 @@ ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) -# ADD_SUBDIRECTORY(vnode) +ADD_SUBDIRECTORY(vnode) # ADD_SUBDIRECTORY(dnode) #ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index f004380336..cc13ab2eca 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -19,11 +19,51 @@ #include #include -// #include "schema.h" +#include "taosdef.h" + #ifdef __cplusplus extern "C" { #endif + +// ----------------- TSDB COLUMN DEFINITION +typedef struct { + int8_t type; // Column type + int16_t colId; // column ID + int32_t bytes; // column bytes + int32_t offset; // point offset in a row data +} STColumn; + +#define colType(col) ((col)->type) +#define colColId(col) ((col)->colId) +#define colBytes(col) ((col)->bytes) +#define colOffset(col) ((col)->offset) + +#define colSetType(col, t) (colType(col) = (t)) +#define colSetColId(col, id) (colColId(col) = (id)) +#define colSetBytes(col, b) (colBytes(col) = (b)) +#define colSetOffset(col, o) (colOffset(col) = (o)) + +STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes); +void tdFreeCol(STColumn *pCol); +void tdColCpy(STColumn *dst, STColumn *src); +void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes); + +// ----------------- TSDB SCHEMA DEFINITION +typedef struct { + int32_t numOfCols; + int32_t padding; // TODO: replace the padding for useful variable + STColumn columns[]; +} STSchema; + +#define schemaNCols(s) ((s)->numOfCols) +#define schemaColAt(s, i) ((s)->columns + i) + +STSchema *tdNewSchema(int32_t nCols); +STSchema *tdDupSchema(STSchema *pSchema); +void tdFreeSchema(STSchema *pSchema); +void tdUpdateSchema(STSchema *pSchema); + // ----------------- Data row structure /* A data row, the format is like below: diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index fe868f91d5..064cb3ff29 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -1,5 +1,135 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include "dataformat.h" +/** + * Create a new STColumn object + * ASSUMPTIONS: VALID PARAMETERS + * + * @param type column type + * @param colId column ID + * @param bytes maximum bytes the col taken + * + * @return a STColumn object on success + * NULL for failure + */ +STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes) { + if (!isValidDataType(type, 0)) return NULL; + + STColumn *pCol = (STColumn *)calloc(1, sizeof(STColumn)); + if (pCol == NULL) return NULL; + + colSetType(pCol, type); + colSetColId(pCol, colId); + colSetOffset(pCol, -1); + switch (type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + colSetBytes(pCol, bytes); + break; + default: + colSetBytes(pCol, TYPE_BYTES[type]); + break; + } + + return pCol; +} + +/** + * Free a STColumn object CREATED with tdNewCol + */ +void tdFreeCol(STColumn *pCol) { + if (pCol) free(pCol); +} + +/** + * Copy from source to destinition + */ +void tdColCpy(STColumn *dst, STColumn *src) { memcpy((void *)dst, (void *)src, sizeof(STColumn)); } + +/** + * Set the column + */ +void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) { + colSetType(pCol, type); + colSetColId(pCol, colId); + switch (type) + { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + colSetBytes(pCol, bytes); + break; + default: + colSetBytes(pCol, TYPE_BYTES[type]); + break; + } +} + +/** + * Create a SSchema object with nCols columns + * ASSUMPTIONS: VALID PARAMETERS + * + * @param nCols number of columns the schema has + * + * @return a STSchema object for success + * NULL for failure + */ +STSchema *tdNewSchema(int32_t nCols) { + int32_t size = sizeof(STSchema) + sizeof(STColumn) * nCols; + + STSchema *pSchema = (STSchema *)calloc(1, size); + if (pSchema == NULL) return NULL; + pSchema->numOfCols = nCols; + + return pSchema; +} + +/** + * Duplicate the schema and return a new object + */ +STSchema *tdDupSchema(STSchema *pSchema) { + STSchema *tSchema = tdNewSchema(schemaNCols(pSchema)); + if (tSchema == NULL) return NULL; + + int32_t size = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); + memcpy((void *)tSchema, (void *)pSchema, size); + + return tSchema; +} + +/** + * Free the SSchema object created by tdNewSchema or tdDupSchema + */ +void tdFreeSchema(STSchema *pSchema) { + if (pSchema == NULL) free(pSchema); +} + +/** + * Function to update each columns's offset field in the schema. + * ASSUMPTIONS: VALID PARAMETERS + */ +void tdUpdateSchema(STSchema *pSchema) { + STColumn *pCol = NULL; + int32_t offset = 0; + for (int i = 0; i < schemaNCols(pSchema); i++) { + pCol = schemaColAt(pSchema, i); + colSetOffset(pCol, offset); + offset += TYPE_BYTES[pCol->type]; + } +} + /** * Create a data row with maximum row length bytes. * diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 0d4ca30e35..14b4d593fb 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -16,7 +16,20 @@ #include "taosdef.h" #include "ttokendef.h" -// #include "tutil.h" + +const int32_t TYPE_BYTES[11] = { + -1, // TSDB_DATA_TYPE_NULL + sizeof(int8_t), // TSDB_DATA_TYPE_BOOL + sizeof(int8_t), // TSDB_DATA_TYPE_TINYINT + sizeof(int16_t), // TSDB_DATA_TYPE_SMALLINT + sizeof(int32_t), // TSDB_DATA_TYPE_INT + sizeof(int64_t), // TSDB_DATA_TYPE_BIGINT + sizeof(float), // TSDB_DATA_TYPE_FLOAT + sizeof(double), // TSDB_DATA_TYPE_DOUBLE + -1, // TSDB_DATA_TYPE_BINARY + sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP + -1 // TSDB_DATA_TYPE_NCHAR +}; tDataTypeDescriptor tDataTypeDesc[11] = { {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE"}, diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 8147f6b9bc..85bdfea98f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -45,6 +45,8 @@ extern "C" { #define TSDB_DATA_TYPE_NCHAR 10 // unicode string // Bytes for each type. +extern const int32_t TYPE_BYTES[11]; +// TODO: replace and remove code below #define CHAR_BYTES sizeof(char) #define SHORT_BYTES sizeof(short) #define INT_BYTES sizeof(int) diff --git a/src/vnode/common/inc/schema.h b/src/vnode/common/inc/schema.h deleted file mode 100644 index 97d177ecff..0000000000 --- a/src/vnode/common/inc/schema.h +++ /dev/null @@ -1,68 +0,0 @@ -#ifndef _TD_SCHEMA_H_ -#define _TD_SCHEMA_H_ - -#include -#include - -#include "type.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// ---- Column definition and operations -typedef struct { - int8_t type; // Column type - int16_t colId; // column ID - int16_t bytes; // column bytes - int32_t offset; // point offset in a row data -} SColumn; - -#define colType(col) ((col)->type) -#define colColId(col) ((col)->colId) -#define colBytes(col) ((col)->bytes) -#define colOffset(col) ((col)->offset) - -#define colSetType(col, t) (colType(col) = (t)) -#define colSetColId(col, id) (colColId(col) = (id)) -#define colSetBytes(col, b) (colBytes(col) = (b)) -#define colSetOffset(col, o) (colOffset(col) = (o)) - -SColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes); -void tdFreeCol(SColumn *pCol); -void tdColCpy(SColumn *dst, SColumn *src); - -// ---- Schema definition and operations -typedef struct { - int32_t numOfCols; - int32_t padding; // TODO: replace the padding for useful variable - SColumn columns[]; -} SSchema; - -#define schemaNCols(s) ((s)->numOfCols) -#define schemaColAt(s, i) ((s)->columns + i) - -SSchema *tdNewSchema(int32_t nCols); -SSchema *tdDupSchema(SSchema *pSchema); -void tdFreeSchema(SSchema *pSchema); -void tdUpdateSchema(SSchema *pSchema); -int32_t tdMaxRowDataBytes(SSchema *pSchema); - -// ---- Inline schema definition and operations - -/* Inline schema definition - * +---------+---------+---------+-----+---------+-----------+-----+-----------+ - * | int32_t | | | | | | | | - * +---------+---------+---------+-----+---------+-----------+-----+-----------+ - * | len | SSchema | SColumn | ... | SColumn | col1_name | ... | colN_name | - * +---------+---------+---------+-----+---------+-----------+-----+-----------+ - */ -typedef char *SISchema; - -// TODO: add operations on SISchema - -#ifdef __cplusplus -} -#endif - -#endif // _TD_SCHEMA_H_ diff --git a/src/vnode/common/inc/type.h b/src/vnode/common/inc/type.h deleted file mode 100644 index 4ca80cee33..0000000000 --- a/src/vnode/common/inc/type.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef _TD_TYPE_H_ -#define _TD_TYPE_H_ - -#include - -typedef enum { - TD_DATATYPE_BOOL = 0, - TD_DATATYPE_TINYINT, - TD_DATATYPE_SMALLINT, - TD_DATATYPE_INT, - TD_DATATYPE_BIGINT, - TD_DATATYPE_FLOAT, - TD_DATATYPE_DOUBLE, - TD_DATATYPE_TIMESTAMP, - TD_DATATYPE_VARCHAR, - TD_DATATYPE_NCHAR, - TD_DATATYPE_BINARY -} td_datatype_t; - -extern const int32_t rowDataLen[]; - -// TODO: finish below -#define TD_DATATYPE_BOOL_NULL -#define TD_DATATYPE_TINYINT_NULL -#define TD_DATATYPE_SMALLINT_NULL -#define TD_DATATYPE_INT_NULL -#define TD_DATATYPE_BIGINT_NULL -#define TD_DATATYPE_FLOAT_NULL -#define TD_DATATYPE_DOUBLE_NULL -#define TD_DATATYPE_TIMESTAMP_NULL -#define TD_DATATYPE_VARCHAR_NULL -#define TD_DATATYPE_NCHAR_NULL -#define TD_DATATYPE_BINARY_NULL - -#define TD_IS_VALID_DATATYPE(type) (((type) > TD_DATA_TYPE_INVLD) && ((type) <= TD_DATATYPE_BINARY)) - -#endif // _TD_TYPE_H_ diff --git a/src/vnode/common/src/schema.c b/src/vnode/common/src/schema.c deleted file mode 100644 index be7007ca8b..0000000000 --- a/src/vnode/common/src/schema.c +++ /dev/null @@ -1,136 +0,0 @@ -#include - -#include "schema.h" - -const int32_t rowDataLen[] = { - sizeof(int8_t), // TD_DATATYPE_BOOL, - sizeof(int8_t), // TD_DATATYPE_TINYINT, - sizeof(int16_t), // TD_DATATYPE_SMALLINT, - sizeof(int32_t), // TD_DATATYPE_INT, - sizeof(int64_t), // TD_DATATYPE_BIGINT, - sizeof(float), // TD_DATATYPE_FLOAT, - sizeof(double), // TD_DATATYPE_DOUBLE, - sizeof(int64_t), // TD_DATATYPE_TIMESTAMP - sizeof(int32_t), // TD_DATATYPE_VARCHAR, - sizeof(int32_t), // TD_DATATYPE_NCHAR, - sizeof(int32_t) // TD_DATATYPE_BINARY -}; - -/** - * Create a new SColumn object - * ASSUMPTIONS: VALID PARAMETERS - * - * @param type column type - * @param colId column ID - * @param bytes maximum bytes the col taken - * - * @return a SColumn object on success - * NULL for failure - */ -SColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes) { - SColumn *pCol = (SColumn *)calloc(1, sizeof(SColumn)); - if (pCol == NULL) return NULL; - - colSetType(pCol, type); - colSetColId(pCol, colId); - switch (type) { - case TD_DATATYPE_VARCHAR: - case TD_DATATYPE_NCHAR: - case TD_DATATYPE_BINARY: - colSetBytes(pCol, bytes); - break; - default: - colSetBytes(pCol, rowDataLen[type]); - break; - } - - return pCol; -} - -/** - * Free a SColumn object CREATED with tdNewCol - */ -void tdFreeCol(SColumn *pCol) { - if (pCol) free(pCol); -} - -void tdColCpy(SColumn *dst, SColumn *src) { memcpy((void *)dst, (void *)src, sizeof(SColumn)); } - -/** - * Create a SSchema object with nCols columns - * ASSUMPTIONS: VALID PARAMETERS - * - * @param nCols number of columns the schema has - * - * @return a SSchema object for success - * NULL for failure - */ -SSchema *tdNewSchema(int32_t nCols) { - int32_t size = sizeof(SSchema) + sizeof(SColumn) * nCols; - - SSchema *pSchema = (SSchema *)calloc(1, size); - if (pSchema == NULL) return NULL; - pSchema->numOfCols = nCols; - - return pSchema; -} - -/** - * Free the SSchema object created by tdNewSchema or tdDupSchema - */ -void tdFreeSchema(SSchema *pSchema) { - if (pSchema == NULL) free(pSchema); -} - -SSchema *tdDupSchema(SSchema *pSchema) { - SSchema *tSchema = tdNewSchema(schemaNCols(pSchema)); - if (tSchema == NULL) return NULL; - - int32_t size = sizeof(SSchema) + sizeof(SColumn) * schemaNCols(pSchema); - memcpy((void *)tSchema, (void *)pSchema, size); - - return tSchema; -} - -/** - * Function to update each columns's offset field in the schema. - * ASSUMPTIONS: VALID PARAMETERS - */ -void tdUpdateSchema(SSchema *pSchema) { - SColumn *pCol = NULL; - int32_t offset = 0; - for (int i = 0; i < schemaNCols(pSchema); i++) { - pCol = schemaColAt(pSchema, i); - colSetOffset(pCol, offset); - offset += rowDataLen[pCol->type]; - } -} - -/** - * Get the maximum size of a row data with the schema - */ -int32_t tdMaxRowDataBytes(SSchema *pSchema) { - int32_t size = 0; - SColumn *pCol = NULL; - for (int i = 0; i < schemaNCols(pSchema); i++) { - pCol = schemaColAt(pSchema, i); - size += rowDataLen[pCol->type]; - - switch (pCol->type) { - case TD_DATATYPE_VARCHAR: - size += (pCol->bytes + 1); // TODO: remove literal here - break; - case TD_DATATYPE_NCHAR: - size += (pCol->bytes + 4); // TODO: check and remove literal here - break; - case TD_DATATYPE_BINARY: - size += pCol->bytes; - break; - - default: - break; - } - } - - return size; -} \ No newline at end of file diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index c8d5f600d9..ecc5dc86b1 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -19,8 +19,8 @@ #include #include +#include "taosdef.h" #include "dataformat.h" -#include "schema.h" #ifdef __cplusplus extern "C" { @@ -83,7 +83,7 @@ typedef struct { int64_t createdTime; int32_t numOfCols; // number of columns. For table form super table, not includes the tag schema - SSchema *schema; // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL + STSchema *schema; // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL // If numOfCols < schema->numOfCols, it is a table created from super table // assert(numOfCols <= schema->numOfCols); diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index 12edfece8c..efab26e1db 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -53,7 +53,7 @@ typedef struct STable { // For TSDB_SUPER_TABLE, it is the schema including tags // For TSDB_NTABLE, it is only the schema, not including tags // For TSDB_STABLE, it is NULL - SSchema *pSchema; + STSchema *pSchema; // Tag value for this table // For TSDB_SUPER_TABLE and TSDB_NTABLE, it is NULL @@ -97,7 +97,7 @@ typedef struct { #define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData) #define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex) -SSchema *tsdbGetTableSchema(STable *pTable); +STSchema *tsdbGetTableSchema(STable *pTable); // ---- Operation on SMetaHandle #define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 155ad19206..935bf6281c 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -261,14 +261,13 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) { // TODO: need to return the number of data inserted int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) { - STsdbRepo * pRepo = (STsdbRepo *)repo; - SSubmitBlock *pBlock = pMsg->data; + SSubmitBlock *pBlock = (SSubmitBlock *)pMsg->data; for (int i = 0; i < pMsg->numOfTables; i++) { // Loop to deal with the submit message if (tsdbInsertDataToTable(repo, pBlock) < 0) { return -1; } - pBlock = ((char *)pBlock) + sizeof(SSubmitBlock) + pBlock->len; + pBlock = (SSubmitBlock *)(((char *)pBlock) + sizeof(SSubmitBlock) + pBlock->len); } return 0; -- GitLab