未验证 提交 d539af37 编写于 作者: Y Yang Zhao 提交者: GitHub

[TD-10689]<enhance>optimize memory using for arm platform (#8313)

* memory save for cmdline

* optimize memory using for arm platform

* remove unused var

* save random data init mem usage

* free db and stb

* add prepared_rand parameter

* free g_Dbs

* fix

* fix coredump when free

* modify free point
上级 4e07a8fa
......@@ -245,6 +245,7 @@ typedef struct SArguments_S {
uint64_t insert_interval;
uint64_t timestamp_step;
int64_t query_times;
int64_t prepared_rand;
uint32_t interlaceRows;
uint32_t reqPerReq; // num_of_records_per_req
uint64_t max_sql_len;
......@@ -366,7 +367,7 @@ typedef struct SDataBase_S {
bool drop; // 0: use exists, 1: if exists, drop then new create
SDbCfg dbCfg;
uint64_t superTblCount;
SSuperTable superTbls[MAX_SUPER_TABLE_COUNT];
SSuperTable* superTbls;
} SDataBase;
typedef struct SDbs_S {
......@@ -385,12 +386,11 @@ typedef struct SDbs_S {
uint32_t threadCount;
uint32_t threadCountForCreateTbl;
uint32_t dbCount;
SDataBase db[MAX_DB_COUNT];
// statistics
uint64_t totalInsertRows;
uint64_t totalAffectedRows;
SDataBase* db;
} SDbs;
typedef struct SpecifiedQueryInfo_S {
......@@ -593,12 +593,12 @@ static int regexMatch(const char *s, const char *reg, int cflags);
/* ************ Global variables ************ */
int32_t g_randint[MAX_PREPARED_RAND];
uint32_t g_randuint[MAX_PREPARED_RAND];
int64_t g_randbigint[MAX_PREPARED_RAND];
uint64_t g_randubigint[MAX_PREPARED_RAND];
float g_randfloat[MAX_PREPARED_RAND];
double g_randdouble[MAX_PREPARED_RAND];
int32_t* g_randint;
uint32_t* g_randuint;
int64_t* g_randbigint;
uint64_t* g_randubigint;
float* g_randfloat;
double* g_randdouble;
char *g_randbool_buff = NULL;
char *g_randint_buff = NULL;
......@@ -666,6 +666,7 @@ SArguments g_args = {
0, // insert_interval
DEFAULT_TIMESTAMP_STEP, // timestamp_step
1, // query_times
10000, // prepared_rand
DEFAULT_INTERLACE_ROWS, // interlaceRows;
30000, // reqPerReq
(1024*1024), // max_sql_len
......@@ -2106,7 +2107,7 @@ static void tmfclose(FILE *fp) {
}
}
static void tmfree(char *buf) {
static void tmfree(void *buf) {
if (NULL != buf) {
free(buf);
buf = NULL;
......@@ -2230,157 +2231,157 @@ static void selectAndGetResult(
static char *rand_bool_str() {
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randbool_buff + ((cursor % MAX_PREPARED_RAND) * BOOL_BUFF_LEN);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randbool_buff + ((cursor % g_args.prepared_rand) * BOOL_BUFF_LEN);
}
static int32_t rand_bool() {
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randint[cursor % MAX_PREPARED_RAND] % 2;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randint[cursor % g_args.prepared_rand] % 2;
}
static char *rand_tinyint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randtinyint_buff +
((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * TINYINT_BUFF_LEN);
}
static int32_t rand_tinyint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randint[cursor % MAX_PREPARED_RAND] % 128;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randint[cursor % g_args.prepared_rand] % 128;
}
static char *rand_utinyint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randutinyint_buff +
((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * TINYINT_BUFF_LEN);
}
static int32_t rand_utinyint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randuint[cursor % MAX_PREPARED_RAND] % 255;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randuint[cursor % g_args.prepared_rand] % 255;
}
static char *rand_smallint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randsmallint_buff +
((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * SMALLINT_BUFF_LEN);
}
static int32_t rand_smallint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randint[cursor % MAX_PREPARED_RAND] % 32768;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randint[cursor % g_args.prepared_rand] % 32768;
}
static char *rand_usmallint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randusmallint_buff +
((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * SMALLINT_BUFF_LEN);
}
static int32_t rand_usmallint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randuint[cursor % MAX_PREPARED_RAND] % 65535;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randuint[cursor % g_args.prepared_rand] % 65535;
}
static char *rand_int_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randint_buff + ((cursor % g_args.prepared_rand) * INT_BUFF_LEN);
}
static int32_t rand_int()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randint[cursor % MAX_PREPARED_RAND];
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randint[cursor % g_args.prepared_rand];
}
static char *rand_uint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randuint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randuint_buff + ((cursor % g_args.prepared_rand) * INT_BUFF_LEN);
}
static int32_t rand_uint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randuint[cursor % MAX_PREPARED_RAND];
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randuint[cursor % g_args.prepared_rand];
}
static char *rand_bigint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randbigint_buff +
((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * BIGINT_BUFF_LEN);
}
static int64_t rand_bigint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randbigint[cursor % MAX_PREPARED_RAND];
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randbigint[cursor % g_args.prepared_rand];
}
static char *rand_ubigint_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randubigint_buff +
((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN);
((cursor % g_args.prepared_rand) * BIGINT_BUFF_LEN);
}
static int64_t rand_ubigint()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randubigint[cursor % MAX_PREPARED_RAND];
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randubigint[cursor % g_args.prepared_rand];
}
static char *rand_float_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randfloat_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randfloat_buff + ((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN);
}
......@@ -2388,58 +2389,58 @@ static float rand_float()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_randfloat[cursor % MAX_PREPARED_RAND];
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randfloat[cursor % g_args.prepared_rand];
}
static char *demo_current_float_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_rand_current_buff +
((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN);
((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN);
}
static float UNUSED_FUNC demo_current_float()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return (float)(9.8 + 0.04 * (g_randint[cursor % MAX_PREPARED_RAND] % 10)
+ g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return (float)(9.8 + 0.04 * (g_randint[cursor % g_args.prepared_rand] % 10)
+ g_randfloat[cursor % g_args.prepared_rand]/1000000000);
}
static char *demo_voltage_int_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_rand_voltage_buff +
((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN);
((cursor % g_args.prepared_rand) * INT_BUFF_LEN);
}
static int32_t UNUSED_FUNC demo_voltage_int()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return 215 + g_randint[cursor % MAX_PREPARED_RAND] % 10;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return 215 + g_randint[cursor % g_args.prepared_rand] % 10;
}
static char *demo_phase_float_str() {
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return g_rand_phase_buff + ((cursor % MAX_PREPARED_RAND) * FLOAT_BUFF_LEN);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_rand_phase_buff + ((cursor % g_args.prepared_rand) * FLOAT_BUFF_LEN);
}
static float UNUSED_FUNC demo_phase_float() {
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
return (float)((115 + g_randint[cursor % MAX_PREPARED_RAND] % 10
+ g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000)/360);
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return (float)((115 + g_randint[cursor % g_args.prepared_rand] % 10
+ g_randfloat[cursor % g_args.prepared_rand]/1000000000)/360);
}
#if 0
......@@ -2478,7 +2479,7 @@ static char *rand_double_str()
{
static int cursor;
cursor++;
if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0;
if (cursor > (g_args.prepared_rand - 1)) cursor = 0;
return g_randdouble_buff + (cursor * DOUBLE_BUFF_LEN);
}
......@@ -2486,42 +2487,54 @@ static double rand_double()
{
static int cursor;
cursor++;
cursor = cursor % MAX_PREPARED_RAND;
cursor = cursor % g_args.prepared_rand;
return g_randdouble[cursor];
}
static void init_rand_data() {
g_randint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND);
g_randint_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand);
assert(g_randint_buff);
g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND);
g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand);
assert(g_rand_voltage_buff);
g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randbigint_buff);
g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randsmallint_buff);
g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randtinyint_buff);
g_randbool_buff = calloc(1, BOOL_BUFF_LEN * MAX_PREPARED_RAND);
g_randbool_buff = calloc(1, BOOL_BUFF_LEN * g_args.prepared_rand);
assert(g_randbool_buff);
g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND);
g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand);
assert(g_randfloat_buff);
g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND);
g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand);
assert(g_rand_current_buff);
g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND);
g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * g_args.prepared_rand);
assert(g_rand_phase_buff);
g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * MAX_PREPARED_RAND);
g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * g_args.prepared_rand);
assert(g_randdouble_buff);
g_randuint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND);
g_randuint_buff = calloc(1, INT_BUFF_LEN * g_args.prepared_rand);
assert(g_randuint_buff);
g_randutinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randutinyint_buff = calloc(1, TINYINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randutinyint_buff);
g_randusmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randusmallint_buff = calloc(1, SMALLINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randusmallint_buff);
g_randubigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND);
g_randubigint_buff = calloc(1, BIGINT_BUFF_LEN * g_args.prepared_rand);
assert(g_randubigint_buff);
for (int i = 0; i < MAX_PREPARED_RAND; i++) {
g_randint = calloc(1, sizeof(int32_t) * g_args.prepared_rand);
assert(g_randint);
g_randuint = calloc(1, sizeof(uint32_t) * g_args.prepared_rand);
assert(g_randuint);
g_randbigint = calloc(1, sizeof(int64_t) * g_args.prepared_rand);
assert(g_randbigint);
g_randubigint = calloc(1, sizeof(uint64_t) * g_args.prepared_rand);
assert(g_randubigint);
g_randfloat = calloc(1, sizeof(float) * g_args.prepared_rand);
assert(g_randfloat);
g_randdouble = calloc(1, sizeof(double) * g_args.prepared_rand);
assert(g_randdouble);
for (int i = 0; i < g_args.prepared_rand; i++) {
g_randint[i] = (int)(taosRandom() % RAND_MAX - (RAND_MAX >> 1));
g_randuint[i] = (int)(taosRandom());
sprintf(g_randint_buff + i * INT_BUFF_LEN, "%d",
......@@ -5267,6 +5280,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
goto PARSE_OVER;
}
cJSON* prepareRand = cJSON_GetObjectItem(root, "prepared_rand");
if (prepareRand && prepareRand->type == cJSON_Number) {
if (prepareRand->valueint <= 0) {
errorPrint("%s() LN%d, failed to read json, prepared_rand input mistake\n",
__func__, __LINE__);
goto PARSE_OVER;
}
g_args.prepared_rand = prepareRand->valueint;
} else if (!prepareRand) {
g_args.prepared_rand = 10000;
} else {
errorPrint("%s() LN%d, failed to read json, prepared_rand not found\n",
__func__, __LINE__);
goto PARSE_OVER;
}
cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
if (answerPrompt
&& answerPrompt->type == cJSON_String
......@@ -5308,7 +5337,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
MAX_DB_COUNT);
goto PARSE_OVER;
}
g_Dbs.db = calloc(1, sizeof(SDataBase)*dbSize);
assert(g_Dbs.db);
g_Dbs.dbCount = dbSize;
for (int i = 0; i < dbSize; ++i) {
cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
......@@ -5508,7 +5538,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
MAX_SUPER_TABLE_COUNT);
goto PARSE_OVER;
}
g_Dbs.db[i].superTbls = calloc(1, stbSize * sizeof(SSuperTable));
assert(g_Dbs.db[i].superTbls);
g_Dbs.db[i].superTblCount = stbSize;
for (int j = 0; j < stbSize; ++j) {
cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
......@@ -6399,9 +6430,12 @@ static bool getInfoFromJsonFile(char* file) {
}
if (INSERT_TEST == g_args.test_mode) {
memset(&g_Dbs, 0, sizeof(SDbs));
g_Dbs.use_metric = g_args.use_metric;
ret = getMetaFromInsertJsonFile(root);
} else if ((QUERY_TEST == g_args.test_mode)
|| (SUBSCRIBE_TEST == g_args.test_mode)) {
memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
ret = getMetaFromQueryJsonFile(root);
} else {
errorPrint("%s",
......@@ -6466,8 +6500,9 @@ static void postFreeResource() {
g_Dbs.db[i].superTbls[j].childTblName = NULL;
}
}
tmfree(g_Dbs.db[i].superTbls);
}
tmfree(g_Dbs.db);
tmfree(g_randbool_buff);
tmfree(g_randint_buff);
tmfree(g_rand_voltage_buff);
......@@ -6490,6 +6525,7 @@ static void postFreeResource() {
}
}
tmfree(g_sampleBindBatchArray);
#endif
}
......@@ -11919,29 +11955,6 @@ static int subscribeTestProcess() {
return 0;
}
static void initOfInsertMeta() {
memset(&g_Dbs, 0, sizeof(SDbs));
// set default values
tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
g_Dbs.port = 6030;
tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN);
g_Dbs.threadCount = 2;
g_Dbs.use_metric = g_args.use_metric;
}
static void initOfQueryMeta() {
memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
// set default values
tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
g_queryInfo.port = 6030;
tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN);
}
static void setParaFromArg() {
char type[20];
char length[20];
......@@ -11974,7 +11987,7 @@ static void setParaFromArg() {
tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
g_Dbs.use_metric = g_args.use_metric;
g_args.prepared_rand = min(g_args.insertRows, MAX_PREPARED_RAND);
g_Dbs.aggr_func = g_args.aggr_func;
char dataString[TSDB_MAX_BYTES_PER_ROW];
......@@ -12251,8 +12264,6 @@ int main(int argc, char *argv[]) {
if (g_args.metaFile) {
g_totalChildTables = 0;
initOfInsertMeta();
initOfQueryMeta();
if (false == getInfoFromJsonFile(g_args.metaFile)) {
printf("Failed to read %s\n", g_args.metaFile);
......@@ -12262,6 +12273,10 @@ int main(int argc, char *argv[]) {
testMetaFile();
} else {
memset(&g_Dbs, 0, sizeof(SDbs));
g_Dbs.db = calloc(1, sizeof(SDataBase));
assert(g_Dbs.db);
g_Dbs.db[0].superTbls = calloc(1, sizeof(SSuperTable));
assert(g_Dbs.db[0].superTbls);
setParaFromArg();
if (NULL != g_args.sqlFile) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册