mndVgroup.c 89.1 KB
Newer Older
H
refact  
Hongze Cheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

S
Shengliang Guan 已提交
16
#define _DEFAULT_SOURCE
S
Shengliang Guan 已提交
17
#include "mndVgroup.h"
S
Shengliang Guan 已提交
18
#include "mndDb.h"
S
Shengliang Guan 已提交
19
#include "mndDnode.h"
S
Shengliang Guan 已提交
20
#include "mndMnode.h"
21
#include "mndPrivilege.h"
S
Shengliang Guan 已提交
22 23
#include "mndShow.h"
#include "mndTrans.h"
S
Shengliang Guan 已提交
24
#include "mndUser.h"
H
Haojun Liao 已提交
25
#include "tmisce.h"
C
cadem 已提交
26
#include "mndStb.h"
S
Shengliang Guan 已提交
27

S
Shengliang Guan 已提交
28 29
#define VGROUP_VER_NUMBER   1
#define VGROUP_RESERVE_SIZE 64
S
Shengliang Guan 已提交
30

S
Shengliang Guan 已提交
31 32 33
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
static int32_t  mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t  mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
S
Shengliang Guan 已提交
34
static int32_t  mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
S
Shengliang Guan 已提交
35

S
Shengliang Guan 已提交
36
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
S
Shengliang Guan 已提交
37
static void    mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
S
Shengliang Guan 已提交
38
static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
S
Shengliang Guan 已提交
39 40
static void    mndCancelGetNextVnode(SMnode *pMnode, void *pIter);

S
Shengliang Guan 已提交
41 42
static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq);
static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq);
S
Shengliang Guan 已提交
43
static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq);
C
cadem 已提交
44
static int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq);
S
Shengliang Guan 已提交
45

S
Shengliang Guan 已提交
46
int32_t mndInitVgroup(SMnode *pMnode) {
S
Shengliang Guan 已提交
47 48 49 50 51 52 53 54 55
  SSdbTable table = {
      .sdbType = SDB_VGROUP,
      .keyType = SDB_KEY_INT32,
      .encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
      .decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
      .insertFp = (SdbInsertFp)mndVgroupActionInsert,
      .updateFp = (SdbUpdateFp)mndVgroupActionUpdate,
      .deleteFp = (SdbDeleteFp)mndVgroupActionDelete,
  };
S
Shengliang Guan 已提交
56

57 58 59 60
  mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndTransProcessRsp);
  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp);
  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp);
  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp);
S
Shengliang Guan 已提交
61
  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_HASHRANGE_RSP, mndTransProcessRsp);
62 63
  mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp);
  mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp);
64
  mndSetMsgHandle(pMnode, TDMT_VND_DISABLE_WRITE_RSP, mndTransProcessRsp);
C
cadem 已提交
65
  mndSetMsgHandle(pMnode, TDMT_SYNC_FORCE_FOLLOWER_RSP, mndTransProcessRsp);
C
cadem 已提交
66
  mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_TYPE_RSP, mndTransProcessRsp);
S
Shengliang Guan 已提交
67

68
  mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg);
S
Shengliang Guan 已提交
69
  mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg);
C
cadem 已提交
70
  //mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessVgroupBalanceLeaderMsg);
71
  mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg);
C
cadem 已提交
72
  mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP_LEADER, mndProcessVgroupBalanceLeaderMsg);
73

S
Shengliang Guan 已提交
74 75
  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups);
  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup);
S
Shengliang Guan 已提交
76 77 78
  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndRetrieveVnodes);
  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndCancelGetNextVnode);

S
Shengliang Guan 已提交
79
  return sdbSetTable(pMnode->pSdb, table);
S
Shengliang Guan 已提交
80 81 82 83
}

void mndCleanupVgroup(SMnode *pMnode) {}

S
Shengliang Guan 已提交
84
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
85 86
  terrno = TSDB_CODE_OUT_OF_MEMORY;

S
Shengliang Guan 已提交
87
  SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE);
S
Shengliang Guan 已提交
88
  if (pRaw == NULL) goto _OVER;
S
Shengliang Guan 已提交
89 90

  int32_t dataPos = 0;
S
Shengliang Guan 已提交
91 92 93 94 95 96 97 98
  SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER)
  SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER)
  SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER)
  SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER)
  SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER)
  SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER)
S
Shengliang Guan 已提交
99
  SDB_SET_INT8(pRaw, dataPos, pVgroup->isTsma, _OVER)
S
Shengliang Guan 已提交
100
  SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER)
S
Shengliang Guan 已提交
101 102
  for (int8_t i = 0; i < pVgroup->replica; ++i) {
    SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
S
Shengliang Guan 已提交
103
    SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER)
104
  }
S
Shengliang Guan 已提交
105
  SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER)
S
Shengliang Guan 已提交
106
  SDB_SET_DATALEN(pRaw, dataPos, _OVER)
107 108 109

  terrno = 0;

S
Shengliang Guan 已提交
110
_OVER:
111 112 113 114
  if (terrno != 0) {
    mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr());
    sdbFreeRaw(pRaw);
    return NULL;
S
Shengliang Guan 已提交
115 116
  }

117
  mTrace("vgId:%d, encode to raw:%p, row:%p", pVgroup->vgId, pRaw, pVgroup);
S
Shengliang Guan 已提交
118 119 120
  return pRaw;
}

S
Shengliang Guan 已提交
121
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
122
  terrno = TSDB_CODE_OUT_OF_MEMORY;
123 124
  SSdbRow *pRow = NULL;
  SVgObj  *pVgroup = NULL;
125

S
Shengliang Guan 已提交
126
  int8_t sver = 0;
S
Shengliang Guan 已提交
127
  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
S
Shengliang Guan 已提交
128

S
Shengliang Guan 已提交
129
  if (sver != VGROUP_VER_NUMBER) {
S
Shengliang Guan 已提交
130
    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
S
Shengliang Guan 已提交
131
    goto _OVER;
S
Shengliang Guan 已提交
132 133
  }

134
  pRow = sdbAllocRow(sizeof(SVgObj));
S
Shengliang Guan 已提交
135
  if (pRow == NULL) goto _OVER;
136

137
  pVgroup = sdbGetRowObj(pRow);
S
Shengliang Guan 已提交
138
  if (pVgroup == NULL) goto _OVER;
S
Shengliang Guan 已提交
139 140

  int32_t dataPos = 0;
S
Shengliang Guan 已提交
141 142 143 144 145 146 147 148
  SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER)
  SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER)
  SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER)
  SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER)
  SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER)
  SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER)
S
Shengliang Guan 已提交
149
  SDB_GET_INT8(pRaw, dataPos, &pVgroup->isTsma, _OVER)
S
Shengliang Guan 已提交
150
  SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER)
S
Shengliang Guan 已提交
151 152
  for (int8_t i = 0; i < pVgroup->replica; ++i) {
    SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
S
Shengliang Guan 已提交
153
    SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER)
S
Shengliang Guan 已提交
154
    if (pVgroup->replica == 1) {
155
      pVgid->syncState = TAOS_SYNC_STATE_LEADER;
S
Shengliang Guan 已提交
156
    }
S
Shengliang Guan 已提交
157
  }
S
Shengliang Guan 已提交
158
  SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER)
159 160 161

  terrno = 0;

S
Shengliang Guan 已提交
162
_OVER:
163
  if (terrno != 0) {
164
    mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup == NULL ? 0 : pVgroup->vgId, pRaw, terrstr());
wafwerar's avatar
wafwerar 已提交
165
    taosMemoryFreeClear(pRow);
166 167
    return NULL;
  }
S
Shengliang Guan 已提交
168

169
  mTrace("vgId:%d, decode from raw:%p, row:%p", pVgroup->vgId, pRaw, pVgroup);
S
Shengliang Guan 已提交
170 171 172 173
  return pRow;
}

static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) {
174
  mTrace("vgId:%d, perform insert action, row:%p", pVgroup->vgId, pVgroup);
S
Shengliang Guan 已提交
175 176 177 178
  return 0;
}

static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) {
179
  mTrace("vgId:%d, perform delete action, row:%p", pVgroup->vgId, pVgroup);
S
Shengliang Guan 已提交
180 181 182
  return 0;
}

S
Shengliang Guan 已提交
183 184 185 186 187 188 189
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
  mTrace("vgId:%d, perform update action, old row:%p new row:%p", pOld->vgId, pOld, pNew);
  pOld->updateTime = pNew->updateTime;
  pOld->version = pNew->version;
  pOld->hashBegin = pNew->hashBegin;
  pOld->hashEnd = pNew->hashEnd;
  pOld->replica = pNew->replica;
S
Shengliang Guan 已提交
190
  pOld->isTsma = pNew->isTsma;
191 192 193 194 195 196 197
  for (int32_t i = 0; i < pNew->replica; ++i) {
    SVnodeGid *pNewGid = &pNew->vnodeGid[i];
    for (int32_t j = 0; j < pOld->replica; ++j) {
      SVnodeGid *pOldGid = &pOld->vnodeGid[j];
      if (pNewGid->dnodeId == pOldGid->dnodeId) {
        pNewGid->syncState = pOldGid->syncState;
        pNewGid->syncRestore = pOldGid->syncRestore;
198
        pNewGid->syncCanRead = pOldGid->syncCanRead;
199 200 201
      }
    }
  }
202 203 204 205 206 207
  pNew->numOfTables = pOld->numOfTables;
  pNew->numOfTimeSeries = pOld->numOfTimeSeries;
  pNew->totalStorage = pOld->totalStorage;
  pNew->compStorage = pOld->compStorage;
  pNew->pointsWritten = pOld->pointsWritten;
  pNew->compact = pOld->compact;
C
cadem 已提交
208
  memcpy(pOld->vnodeGid, pNew->vnodeGid, (TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA) * sizeof(SVnodeGid));
S
Shengliang Guan 已提交
209 210 211 212
  return 0;
}

SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) {
S
Shengliang Guan 已提交
213 214
  SSdb   *pSdb = pMnode->pSdb;
  SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId);
S
Shengliang Guan 已提交
215
  if (pVgroup == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
S
Shengliang Guan 已提交
216 217 218
    terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
  }
  return pVgroup;
S
Shengliang Guan 已提交
219 220 221 222 223 224 225
}

void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
  SSdb *pSdb = pMnode->pSdb;
  sdbRelease(pSdb, pVgroup);
}

S
Shengliang Guan 已提交
226
void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
S
Shengliang Guan 已提交
227 228 229 230 231
  SCreateVnodeReq createReq = {0};
  createReq.vgId = pVgroup->vgId;
  memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN);
  createReq.dbUid = pDb->uid;
  createReq.vgVersion = pVgroup->version;
S
Shengliang Guan 已提交
232 233 234 235
  createReq.numOfStables = pDb->cfg.numOfStables;
  createReq.buffer = pDb->cfg.buffer;
  createReq.pageSize = pDb->cfg.pageSize;
  createReq.pages = pDb->cfg.pages;
236
  createReq.cacheLastSize = pDb->cfg.cacheLastSize;
S
Shengliang Guan 已提交
237 238 239 240
  createReq.daysPerFile = pDb->cfg.daysPerFile;
  createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
  createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
S
Shengliang Guan 已提交
241 242
  createReq.minRows = pDb->cfg.minRows;
  createReq.maxRows = pDb->cfg.maxRows;
243
  createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
S
Shengliang Guan 已提交
244 245 246
  createReq.walLevel = pDb->cfg.walLevel;
  createReq.precision = pDb->cfg.precision;
  createReq.compression = pDb->cfg.compression;
S
Shengliang Guan 已提交
247
  createReq.strict = pDb->cfg.strict;
248
  createReq.cacheLast = pDb->cfg.cacheLast;
C
cadem 已提交
249 250
  createReq.replica = 0;
  createReq.learnerReplica = 0;
S
Shengliang Guan 已提交
251
  createReq.selfIndex = -1;
C
cadem 已提交
252
  createReq.learnerSelfIndex = -1;
D
dapan1121 已提交
253 254
  createReq.hashBegin = pVgroup->hashBegin;
  createReq.hashEnd = pVgroup->hashEnd;
S
Shengliang Guan 已提交
255
  createReq.hashMethod = pDb->cfg.hashMethod;
S
sma  
Shengliang Guan 已提交
256 257
  createReq.numOfRetensions = pDb->cfg.numOfRetensions;
  createReq.pRetensions = pDb->cfg.pRetensions;
C
Cary Xu 已提交
258 259
  createReq.isTsma = pVgroup->isTsma;
  createReq.pTsma = pVgroup->pTsma;
S
Shengliang Guan 已提交
260 261 262 263
  createReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
  createReq.walRetentionSize = pDb->cfg.walRetentionSize;
  createReq.walRollPeriod = pDb->cfg.walRollPeriod;
  createReq.walSegmentSize = pDb->cfg.walSegmentSize;
264
  createReq.sstTrigger = pDb->cfg.sstTrigger;
265 266
  createReq.hashPrefix = pDb->cfg.hashPrefix;
  createReq.hashSuffix = pDb->cfg.hashSuffix;
267 268
  createReq.tsdbPageSize = pDb->cfg.tsdbPageSize;

S
Shengliang Guan 已提交
269
  for (int32_t v = 0; v < pVgroup->replica; ++v) {
C
cadem 已提交
270 271 272 273 274 275 276 277 278
    SReplica  *pReplica = NULL;

    if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){
      pReplica = &createReq.replicas[createReq.replica];
    }
    else{
      pReplica = &createReq.learnerReplicas[createReq.learnerReplica];
    }

S
Shengliang Guan 已提交
279 280 281 282 283 284
    SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
    SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
    if (pVgidDnode == NULL) {
      return NULL;
    }

S
Shengliang Guan 已提交
285 286
    pReplica->id = pVgidDnode->id;
    pReplica->port = pVgidDnode->port;
S
Shengliang Guan 已提交
287 288 289
    memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN);
    mndReleaseDnode(pMnode, pVgidDnode);

C
cadem 已提交
290 291 292 293 294 295 296 297 298 299 300 301
    if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){
      if (pDnode->id == pVgid->dnodeId) {
        createReq.selfIndex = createReq.replica;
      }
    }
    else{
      if (pDnode->id == pVgid->dnodeId) {
        createReq.learnerSelfIndex = createReq.learnerReplica;
      }
    }

    if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){
H
Haojun Liao 已提交
302
      createReq.replica++;
C
cadem 已提交
303 304 305
    }
    else{
      createReq.learnerReplica++;
S
Shengliang Guan 已提交
306 307 308
    }
  }

C
cadem 已提交
309
  if (createReq.selfIndex == -1 && createReq.learnerSelfIndex == -1) {
S
Shengliang Guan 已提交
310
    terrno = TSDB_CODE_APP_ERROR;
S
Shengliang Guan 已提交
311 312 313
    return NULL;
  }

H
Haojun Liao 已提交
314
  mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d",
C
cadem 已提交
315 316
                          createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica,
                          createReq.learnerReplica, createReq.strict);
S
Shengliang Guan 已提交
317 318 319
  for (int32_t i = 0; i < createReq.replica; ++i) {
    mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.replicas[i].fqdn, createReq.replicas[i].port);
  }
C
cadem 已提交
320
  for (int32_t i = 0; i < createReq.learnerReplica; ++i) {
H
Haojun Liao 已提交
321
    mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.learnerReplicas[i].fqdn,
C
cadem 已提交
322 323
                                                                createReq.learnerReplicas[i].port);
  }
S
Shengliang Guan 已提交
324

S
Shengliang Guan 已提交
325 326 327 328 329 330
  int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq);
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

wafwerar's avatar
wafwerar 已提交
331
  void *pReq = taosMemoryMalloc(contLen);
S
Shengliang Guan 已提交
332 333 334 335 336 337 338 339
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  tSerializeSCreateVnodeReq(pReq, contLen, &createReq);
  *pContLen = contLen;
  return pReq;
S
Shengliang Guan 已提交
340 341
}

S
Shengliang Guan 已提交
342 343
static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
  SAlterVnodeConfigReq alterReq = {0};
S
Shengliang Guan 已提交
344 345 346
  alterReq.vgVersion = pVgroup->version;
  alterReq.buffer = pDb->cfg.buffer;
  alterReq.pageSize = pDb->cfg.pageSize;
347
  alterReq.pages = pDb->cfg.pages;
348
  alterReq.cacheLastSize = pDb->cfg.cacheLastSize;
S
Shengliang Guan 已提交
349 350 351 352
  alterReq.daysPerFile = pDb->cfg.daysPerFile;
  alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
  alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
  alterReq.daysToKeep2 = pDb->cfg.daysToKeep2;
353
  alterReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
S
Shengliang Guan 已提交
354 355
  alterReq.walLevel = pDb->cfg.walLevel;
  alterReq.strict = pDb->cfg.strict;
356
  alterReq.cacheLast = pDb->cfg.cacheLast;
357 358
  alterReq.sttTrigger = pDb->cfg.sstTrigger;
  alterReq.minRows = pDb->cfg.minRows;
359 360
  alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
  alterReq.walRetentionSize = pDb->cfg.walRetentionSize;
S
Shengliang Guan 已提交
361

S
Shengliang Guan 已提交
362
  mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId);
S
Shengliang Guan 已提交
363
  int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq);
S
Shengliang Guan 已提交
364 365 366 367
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }
S
Shengliang Guan 已提交
368
  contLen += sizeof(SMsgHead);
S
Shengliang Guan 已提交
369 370 371 372 373 374 375 376 377 378 379

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  SMsgHead *pHead = pReq;
  pHead->contLen = htonl(contLen);
  pHead->vgId = htonl(pVgroup->vgId);

S
Shengliang Guan 已提交
380
  tSerializeSAlterVnodeConfigReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq);
S
Shengliang Guan 已提交
381 382 383 384
  *pContLen = contLen;
  return pReq;
}

S
Shengliang Guan 已提交
385
static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId,
S
Shengliang Guan 已提交
386
                                          int32_t *pContLen) {
S
Shengliang Guan 已提交
387
  SAlterVnodeReplicaReq alterReq = {
S
Shengliang Guan 已提交
388 389
      .vgId = pVgroup->vgId,
      .strict = pDb->cfg.strict,
C
cadem 已提交
390 391
      .replica = 0,
      .learnerReplica = 0,
S
Shengliang Guan 已提交
392
      .selfIndex = -1,
C
cadem 已提交
393
      .learnerSelfIndex = -1,
S
Shengliang Guan 已提交
394
  };
S
Shengliang Guan 已提交
395 396

  for (int32_t v = 0; v < pVgroup->replica; ++v) {
C
cadem 已提交
397 398 399 400
    SReplica  *pReplica = NULL;

    if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){
      pReplica = &alterReq.replicas[alterReq.replica];
H
Haojun Liao 已提交
401
      alterReq.replica++;
C
cadem 已提交
402 403 404 405 406
    }
    else{
      pReplica = &alterReq.learnerReplicas[alterReq.learnerReplica];
      alterReq.learnerReplica++;
    }
H
Haojun Liao 已提交
407

S
Shengliang Guan 已提交
408 409
    SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
    SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
410
    if (pVgidDnode == NULL) return NULL;
S
Shengliang Guan 已提交
411 412 413 414 415 416

    pReplica->id = pVgidDnode->id;
    pReplica->port = pVgidDnode->port;
    memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN);
    mndReleaseDnode(pMnode, pVgidDnode);

C
cadem 已提交
417 418 419 420 421 422 423 424 425
    if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){
      if (dnodeId == pVgid->dnodeId) {
        alterReq.selfIndex = v;
      }
    }
    else{
      if (dnodeId == pVgid->dnodeId) {
        alterReq.learnerSelfIndex = v;
      }
S
Shengliang Guan 已提交
426 427
    }
  }
C
cadem 已提交
428

H
Haojun Liao 已提交
429 430
  mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d",
                    alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica,
C
cadem 已提交
431
                                                      alterReq.learnerSelfIndex, alterReq.strict);
432 433 434
  for (int32_t i = 0; i < alterReq.replica; ++i) {
    mInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port);
  }
C
cadem 已提交
435
  for (int32_t i = 0; i < alterReq.learnerReplica; ++i) {
H
Haojun Liao 已提交
436
    mInfo("vgId:%d, learnerReplica:%d ep:%s:%u", alterReq.vgId, i,
C
cadem 已提交
437 438
                alterReq.learnerReplicas[i].fqdn, alterReq.learnerReplicas[i].port);
  }
S
Shengliang Guan 已提交
439

C
cadem 已提交
440
  if (alterReq.selfIndex == -1 && alterReq.learnerSelfIndex == -1) {
S
Shengliang Guan 已提交
441
    terrno = TSDB_CODE_APP_ERROR;
S
Shengliang Guan 已提交
442 443
    return NULL;
  }
444

S
Shengliang Guan 已提交
445
  int32_t contLen = tSerializeSAlterVnodeReplicaReq(NULL, 0, &alterReq);
446 447 448 449 450 451 452 453 454 455 456
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

S
Shengliang Guan 已提交
457
  tSerializeSAlterVnodeReplicaReq(pReq, contLen, &alterReq);
458 459 460 461
  *pContLen = contLen;
  return pReq;
}

462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
static void *mndBuildDisableVnodeWriteReq(SMnode *pMnode, SDbObj *pDb, int32_t vgId, int32_t *pContLen) {
  SDisableVnodeWriteReq disableReq = {
      .vgId = vgId,
      .disable = 1,
  };

  mInfo("vgId:%d, build disable vnode write req", vgId);
  int32_t contLen = tSerializeSDisableVnodeWriteReq(NULL, 0, &disableReq);
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  tSerializeSDisableVnodeWriteReq(pReq, contLen, &disableReq);
  *pContLen = contLen;
  return pReq;
}

486 487 488 489 490 491 492 493
static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dstVgId, int32_t *pContLen) {
  SAlterVnodeHashRangeReq alterReq = {
      .srcVgId = pVgroup->vgId,
      .dstVgId = dstVgId,
      .hashBegin = pVgroup->hashBegin,
      .hashEnd = pVgroup->hashEnd,
  };

494
  mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", pVgroup->vgId, dstVgId,
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
        pVgroup->hashBegin, pVgroup->hashEnd);
  int32_t contLen = tSerializeSAlterVnodeHashRangeReq(NULL, 0, &alterReq);
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  tSerializeSAlterVnodeHashRangeReq(pReq, contLen, &alterReq);
  *pContLen = contLen;
  return pReq;
}

L
Liu Jicong 已提交
513
void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
S
Shengliang Guan 已提交
514 515 516 517 518 519
  SDropVnodeReq dropReq = {0};
  dropReq.dnodeId = pDnode->id;
  dropReq.vgId = pVgroup->vgId;
  memcpy(dropReq.db, pDb->name, TSDB_DB_FNAME_LEN);
  dropReq.dbUid = pDb->uid;

S
Shengliang Guan 已提交
520
  mInfo("vgId:%d, build drop vnode req", dropReq.vgId);
S
Shengliang Guan 已提交
521 522
  int32_t contLen = tSerializeSDropVnodeReq(NULL, 0, &dropReq);
  if (contLen < 0) {
S
Shengliang Guan 已提交
523 524 525 526
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

wafwerar's avatar
wafwerar 已提交
527
  void *pReq = taosMemoryMalloc(contLen);
S
Shengliang Guan 已提交
528 529 530 531
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }
S
Shengliang Guan 已提交
532

S
Shengliang Guan 已提交
533 534 535
  tSerializeSDropVnodeReq(pReq, contLen, &dropReq);
  *pContLen = contLen;
  return pReq;
S
Shengliang Guan 已提交
536 537
}

S
Shengliang Guan 已提交
538 539 540
static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
  SDnodeObj *pDnode = pObj;
  pDnode->numOfVnodes = 0;
541
  pDnode->numOfOtherNodes = 0;
S
Shengliang Guan 已提交
542 543 544 545 546 547
  return true;
}

static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
  SDnodeObj *pDnode = pObj;
  SArray    *pArray = p1;
S
Shengliang Guan 已提交
548 549 550 551 552
  int32_t    exceptDnodeId = *(int32_t *)p2;

  if (exceptDnodeId == pDnode->id) {
    return true;
  }
S
Shengliang Guan 已提交
553 554

  int64_t curMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
555
  bool    online = mndIsDnodeOnline(pDnode, curMs);
S
Shengliang Guan 已提交
556 557
  bool    isMnode = mndIsMnode(pMnode, pDnode->id);
  pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
558
  pDnode->memUsed = mndGetVnodesMemory(pMnode, pDnode->id);
S
Shengliang Guan 已提交
559

560
  mInfo("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id,
561
        pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed);
S
Shengliang Guan 已提交
562 563

  if (isMnode) {
564
    pDnode->numOfOtherNodes++;
S
Shengliang Guan 已提交
565 566
  }

S
Shengliang Guan 已提交
567 568 569
  if (online && pDnode->numOfSupportVnodes > 0) {
    taosArrayPush(pArray, pDnode);
  }
S
Shengliang Guan 已提交
570 571 572
  return true;
}

S
Shengliang Guan 已提交
573
SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) {
S
Shengliang Guan 已提交
574
  SSdb   *pSdb = pMnode->pSdb;
S
Shengliang Guan 已提交
575
  int32_t numOfDnodes = mndGetDnodeSize(pMnode);
S
Shengliang Guan 已提交
576

S
Shengliang Guan 已提交
577 578 579 580 581
  SArray *pArray = taosArrayInit(numOfDnodes, sizeof(SDnodeObj));
  if (pArray == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }
S
Shengliang Guan 已提交
582

S
Shengliang Guan 已提交
583
  sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL);
S
Shengliang Guan 已提交
584
  sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL);
585 586 587 588 589 590

  mDebug("build %d dnodes array", (int32_t)taosArrayGetSize(pArray));
  for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
    SDnodeObj *pDnode = taosArrayGet(pArray, i);
    mDebug("dnode:%d, vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes);
  }
S
Shengliang Guan 已提交
591 592 593
  return pArray;
}

sangshuduo's avatar
sangshuduo 已提交
594
static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) {
X
Xiaoyu Wang 已提交
595 596 597 598
  if (*dnode1Id == *dnode2Id) {
    return 0;
  }
  return *dnode1Id > *dnode2Id ? 1 : -1;
sangshuduo's avatar
sangshuduo 已提交
599
}
S
Shengliang Guan 已提交
600

601 602 603
static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) {
  float totalDnodes = pDnode->numOfVnodes + (float)pDnode->numOfOtherNodes * ratio + additionDnodes;
  return totalDnodes / pDnode->numOfSupportVnodes;
604 605
}

S
Shengliang Guan 已提交
606
static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) {
607 608
  float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9);
  float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9);
X
Xiaoyu Wang 已提交
609 610 611 612
  if (d1Score == d2Score) {
    return 0;
  }
  return d1Score > d2Score ? 1 : -1;
S
Shengliang Guan 已提交
613 614
}

S
Shengliang Guan 已提交
615 616 617 618 619 620 621 622 623 624
void mndSortVnodeGid(SVgObj *pVgroup) {
  for (int32_t i = 0; i < pVgroup->replica; ++i) {
    for (int32_t j = 0; j < pVgroup->replica - 1 - i; ++j) {
      if (pVgroup->vnodeGid[j].dnodeId > pVgroup->vnodeGid[j + 1].dnodeId) {
        TSWAP(pVgroup->vnodeGid[j], pVgroup->vnodeGid[j + 1]);
      }
    }
  }
}

625
static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
S
Shengliang Guan 已提交
626 627 628 629
  SSdb   *pSdb = pMnode->pSdb;
  int32_t allocedVnodes = 0;
  void   *pIter = NULL;

630
  mDebug("start to sort %d dnodes", (int32_t)taosArrayGetSize(pArray));
S
Shengliang Guan 已提交
631
  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
632 633
  for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
    SDnodeObj *pDnode = taosArrayGet(pArray, i);
634
    mDebug("dnode:%d, score:%f", pDnode->id, mndGetDnodeScore(pDnode, 0, 0.9));
635
  }
S
Shengliang Guan 已提交
636

S
Shengliang Guan 已提交
637 638 639 640 641 642 643 644
  int32_t size = taosArrayGetSize(pArray);
  if (size < pVgroup->replica) {
    mError("db:%s, vgId:%d, no enough online dnodes:%d to alloc %d replica", pVgroup->dbName, pVgroup->vgId, size,
           pVgroup->replica);
    terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
    return -1;
  }

S
Shengliang Guan 已提交
645 646 647
  for (int32_t v = 0; v < pVgroup->replica; ++v) {
    SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
    SDnodeObj *pDnode = taosArrayGet(pArray, v);
648
    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
S
Shengliang Guan 已提交
649 650 651 652
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      return -1;
    }

653 654 655 656 657 658 659 660 661 662
    int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup);
    if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d, avail:%" PRId64 " used:%" PRId64,
             pVgroup->dbName, pVgroup->vgId, vgMem, pDnode->id, pDnode->memAvail, pDnode->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
      return -1;
    } else {
      pDnode->memUsed += vgMem;
    }

S
Shengliang Guan 已提交
663 664
    pVgid->dnodeId = pDnode->id;
    if (pVgroup->replica == 1) {
665
      pVgid->syncState = TAOS_SYNC_STATE_LEADER;
S
Shengliang Guan 已提交
666
    } else {
667
      pVgid->syncState = TAOS_SYNC_STATE_FOLLOWER;
S
Shengliang Guan 已提交
668 669
    }

670 671
    mInfo("db:%s, vgId:%d, vn:%d is alloced, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64,
          pVgroup->dbName, pVgroup->vgId, v, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed);
S
Shengliang Guan 已提交
672
    pDnode->numOfVnodes++;
S
Shengliang Guan 已提交
673
  }
S
Shengliang Guan 已提交
674

S
Shengliang Guan 已提交
675
  mndSortVnodeGid(pVgroup);
S
Shengliang Guan 已提交
676 677
  return 0;
}
678

S
Shengliang Guan 已提交
679
int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) {
S
Shengliang Guan 已提交
680
  SArray *pArray = mndBuildDnodesArray(pMnode, 0);
S
Shengliang Guan 已提交
681 682 683 684 685 686 687 688 689 690 691
  if (pArray == NULL) return -1;

  pVgroup->vgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
  pVgroup->isTsma = 1;
  pVgroup->createdTime = taosGetTimestampMs();
  pVgroup->updateTime = pVgroup->createdTime;
  pVgroup->version = 1;
  memcpy(pVgroup->dbName, pDb->name, TSDB_DB_FNAME_LEN);
  pVgroup->dbUid = pDb->uid;
  pVgroup->replica = 1;

692
  if (mndGetAvailableDnode(pMnode, pDb, pVgroup, pArray) != 0) return -1;
L
Liu Jicong 已提交
693
  taosArrayDestroy(pArray);
S
Shengliang Guan 已提交
694 695 696 697 698

  mInfo("db:%s, sma vgId:%d is alloced", pDb->name, pVgroup->vgId);
  return 0;
}

S
Shengliang Guan 已提交
699
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
S
Shengliang Guan 已提交
700 701 702 703
  int32_t code = -1;
  SArray *pArray = NULL;
  SVgObj *pVgroups = NULL;

wafwerar's avatar
wafwerar 已提交
704
  pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj));
S
Shengliang Guan 已提交
705 706
  if (pVgroups == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
S
Shengliang Guan 已提交
707
    goto _OVER;
S
Shengliang Guan 已提交
708 709
  }

S
Shengliang Guan 已提交
710
  pArray = mndBuildDnodesArray(pMnode, 0);
S
Shengliang Guan 已提交
711
  if (pArray == NULL) goto _OVER;
S
Shengliang Guan 已提交
712

S
Shengliang Guan 已提交
713 714
  mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray),
        pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications);
S
Shengliang Guan 已提交
715

S
Shengliang Guan 已提交
716
  int32_t  allocedVgroups = 0;
S
Shengliang Guan 已提交
717 718 719
  int32_t  maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
  uint32_t hashMin = 0;
  uint32_t hashMax = UINT32_MAX;
720
  uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups;
S
Shengliang Guan 已提交
721

722 723
  if (maxVgId < 2) maxVgId = 2;

724
  for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) {
S
Shengliang Guan 已提交
725
    SVgObj *pVgroup = &pVgroups[v];
S
Shengliang Guan 已提交
726
    pVgroup->vgId = maxVgId++;
S
Shengliang Guan 已提交
727 728
    pVgroup->createdTime = taosGetTimestampMs();
    pVgroup->updateTime = pVgroups->createdTime;
S
Shengliang Guan 已提交
729
    pVgroup->version = 1;
S
Shengliang Guan 已提交
730
    pVgroup->hashBegin = hashMin + hashInterval * v;
731
    if (v == pDb->cfg.numOfVgroups - 1) {
S
Shengliang Guan 已提交
732 733
      pVgroup->hashEnd = hashMax;
    } else {
S
Shengliang Guan 已提交
734
      pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1;
S
Shengliang Guan 已提交
735
    }
S
Shengliang Guan 已提交
736

737
    memcpy(pVgroup->dbName, pDb->name, TSDB_DB_FNAME_LEN);
S
Shengliang Guan 已提交
738
    pVgroup->dbUid = pDb->uid;
S
Shengliang Guan 已提交
739
    pVgroup->replica = pDb->cfg.replications;
S
Shengliang Guan 已提交
740

741
    if (mndGetAvailableDnode(pMnode, pDb, pVgroup, pArray) != 0) {
S
Shengliang Guan 已提交
742
      goto _OVER;
S
Shengliang Guan 已提交
743 744
    }

S
Shengliang Guan 已提交
745
    allocedVgroups++;
746 747
  }

S
Shengliang Guan 已提交
748
  *ppVgroups = pVgroups;
S
Shengliang Guan 已提交
749 750
  code = 0;

751
  mInfo("db:%s, total %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications);
S
Shengliang Guan 已提交
752

S
Shengliang Guan 已提交
753
_OVER:
wafwerar's avatar
wafwerar 已提交
754
  if (code != 0) taosMemoryFree(pVgroups);
S
Shengliang Guan 已提交
755 756
  taosArrayDestroy(pArray);
  return code;
757
}
758

L
Liu Jicong 已提交
759
SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
760 761 762
  SEpSet epset = {0};

  for (int32_t v = 0; v < pVgroup->replica; ++v) {
L
Liu Jicong 已提交
763 764
    const SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
    SDnodeObj       *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
765 766
    if (pDnode == NULL) continue;

767
    if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
768 769 770
      epset.inUse = epset.numOfEps;
    }

H
Haojun Liao 已提交
771
    addEpIntoEpSet(&epset, pDnode->fqdn, pDnode->port);
772 773 774 775 776 777
    mndReleaseDnode(pMnode, pDnode);
  }

  return epset;
}

S
Shengliang Guan 已提交
778 779
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
  SMnode *pMnode = pReq->info.node;
S
Shengliang Guan 已提交
780 781 782 783
  SSdb   *pSdb = pMnode->pSdb;
  int32_t numOfRows = 0;
  SVgObj *pVgroup = NULL;
  int32_t cols = 0;
784
  int64_t curMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
785

H
Haojun Liao 已提交
786 787 788 789 790 791 792
  SDbObj *pDb = NULL;
  if (strlen(pShow->db) > 0) {
    pDb = mndAcquireDb(pMnode, pShow->db);
    if (pDb == NULL) {
      return 0;
    }
  }
S
Shengliang Guan 已提交
793

S
Shengliang Guan 已提交
794 795 796 797
  while (numOfRows < rows) {
    pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup);
    if (pShow->pIter == NULL) break;

H
Haojun Liao 已提交
798
    if (pDb != NULL && pVgroup->dbUid != pDb->uid) {
799
      sdbRelease(pSdb, pVgroup);
H
Haojun Liao 已提交
800 801 802 803
      continue;
    }

    cols = 0;
804
    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
805
    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
H
Haojun Liao 已提交
806 807

    SName name = {0};
808 809 810 811
    char  db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
    tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB);
    tNameGetDbName(&name, varDataVal(db));
    varDataSetLen(db, strlen(varDataVal(db)));
H
Haojun Liao 已提交
812

813
    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
814
    colDataSetVal(pColInfo, numOfRows, (const char *)db, false);
H
Haojun Liao 已提交
815

816
    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
817
    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false);
H
Haojun Liao 已提交
818

819 820
    // default 3 replica, add 1 replica if move vnode
    for (int32_t i = 0; i < 4; ++i) {
821
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
822
      if (i < pVgroup->replica) {
823
        int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId;
824
        colDataSetVal(pColInfo, numOfRows, (const char *)&dnodeId, false);
825

826
        bool       exist = false;
827 828 829
        bool       online = false;
        SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId);
        if (pDnode != NULL) {
830
          exist = true;
S
Shengliang Guan 已提交
831
          online = mndIsDnodeOnline(pDnode, curMs);
832 833 834
          mndReleaseDnode(pMnode, pDnode);
        }

835 836
        char buf1[20] = {0};
        char role[20] = "offline";
837 838 839
        if (!exist) {
          strcpy(role, "dropping");
        } else if (online) {
840 841 842 843 844 845 846 847 848 849
          char *star = "";
          if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
            if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
              star = "**";
            } else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
              star = "*";
            } else {
            }
          }
          snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), star);
850
        } else {
851
        }
S
Shengliang Guan 已提交
852
        STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
853 854

        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
855
        colDataSetVal(pColInfo, numOfRows, (const char *)buf1, false);
856
      } else {
857
        colDataSetNULL(pColInfo, numOfRows);
858
        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
859
        colDataSetNULL(pColInfo, numOfRows);
860
      }
S
Shengliang Guan 已提交
861
    }
S
Shengliang Guan 已提交
862

863
    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
864
    int32_t cacheUsage = (int32_t)pVgroup->cacheUsage;
865
    colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false);
866

D
dapan1121 已提交
867 868 869
    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfCachedTables, false);

S
Shengliang Guan 已提交
870
    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
871
    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
S
Shengliang Guan 已提交
872

H
Hongze Cheng 已提交
873 874
    // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
    // if (pDb == NULL || pDb->compactStartTime <= 0) {
X
Xiaoyu Wang 已提交
875
    //   colDataSetNULL(pColInfo, numOfRows);
H
Hongze Cheng 已提交
876 877 878
    // } else {
    //   colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false);
    // }
879

H
Haojun Liao 已提交
880
    numOfRows++;
S
Shengliang Guan 已提交
881 882 883
    sdbRelease(pSdb, pVgroup);
  }

H
Haojun Liao 已提交
884 885 886 887
  if (pDb != NULL) {
    mndReleaseDb(pMnode, pDb);
  }

888
  pShow->numOfRows += numOfRows;
S
Shengliang Guan 已提交
889 890 891 892 893 894 895 896
  return numOfRows;
}

static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) {
  SSdb *pSdb = pMnode->pSdb;
  sdbCancelFetch(pSdb, pIter);
}

S
Shengliang Guan 已提交
897 898 899 900
static bool mndGetVnodesNumFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
  SVgObj  *pVgroup = pObj;
  int32_t  dnodeId = *(int32_t *)p1;
  int32_t *pNumOfVnodes = (int32_t *)p2;
S
Shengliang Guan 已提交
901

S
Shengliang Guan 已提交
902 903 904
  for (int32_t v = 0; v < pVgroup->replica; ++v) {
    if (pVgroup->vnodeGid[v].dnodeId == dnodeId) {
      (*pNumOfVnodes)++;
S
Shengliang Guan 已提交
905
    }
S
Shengliang Guan 已提交
906 907
  }

S
Shengliang Guan 已提交
908 909 910 911 912 913
  return true;
}

int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) {
  int32_t numOfVnodes = 0;
  sdbTraverse(pMnode->pSdb, SDB_VGROUP, mndGetVnodesNumFp, &dnodeId, &numOfVnodes, NULL);
S
Shengliang Guan 已提交
914
  return numOfVnodes;
S
Shengliang Guan 已提交
915 916
}

917 918 919 920 921 922
int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDbInput, SVgObj *pVgroup) {
  SDbObj *pDb = pDbInput;
  if (pDbInput == NULL) {
    pDb = mndAcquireDb(pMnode, pVgroup->dbName);
  }

S
Shengliang Guan 已提交
923 924 925
  int64_t vgroupMemroy = 0;
  if (pDb != NULL) {
    vgroupMemroy = (int64_t)pDb->cfg.buffer * 1024 * 1024 + (int64_t)pDb->cfg.pages * pDb->cfg.pageSize * 1024;
926 927
    if (pDb->cfg.cacheLast > 0) {
      vgroupMemroy += (int64_t)pDb->cfg.cacheLastSize * 1024 * 1024;
S
Shengliang Guan 已提交
928
    }
929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956
  }

  if (pDbInput == NULL) {
    mndReleaseDb(pMnode, pDb);
  }
  return vgroupMemroy;
}

static bool mndGetVnodeMemroyFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
  SVgObj  *pVgroup = pObj;
  int32_t  dnodeId = *(int32_t *)p1;
  int64_t *pVnodeMemory = (int64_t *)p2;

  for (int32_t v = 0; v < pVgroup->replica; ++v) {
    if (pVgroup->vnodeGid[v].dnodeId == dnodeId) {
      *pVnodeMemory += mndGetVgroupMemory(pMnode, NULL, pVgroup);
    }
  }

  return true;
}

int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId) {
  int64_t vnodeMemory = 0;
  sdbTraverse(pMnode->pSdb, SDB_VGROUP, mndGetVnodeMemroyFp, &dnodeId, &vnodeMemory, NULL);
  return vnodeMemory;
}

S
Shengliang Guan 已提交
957 958
static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
  SMnode *pMnode = pReq->info.node;
S
Shengliang Guan 已提交
959 960 961 962 963 964 965 966 967
  SSdb   *pSdb = pMnode->pSdb;
  int32_t numOfRows = 0;
  SVgObj *pVgroup = NULL;
  int32_t cols = 0;

  while (numOfRows < rows) {
    pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup);
    if (pShow->pIter == NULL) break;

S
Shengliang Guan 已提交
968
    for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) {
S
Shengliang Guan 已提交
969 970
      SVnodeGid       *pVgid = &pVgroup->vnodeGid[i];
      SColumnInfoData *pColInfo = NULL;
S
Shengliang Guan 已提交
971 972
      cols = 0;

973
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
974
      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
H
Haojun Liao 已提交
975

976
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
977
      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->replica, false);
H
Haojun Liao 已提交
978

979
      char buf[20] = {0};
980
      STR_TO_VARSTR(buf, syncStr(pVgid->syncState));
981
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
982
      colDataSetVal(pColInfo, numOfRows, (const char *)buf, false);
H
Haojun Liao 已提交
983

S
Shengliang Guan 已提交
984 985 986 987 988 989 990 991
      const char *dbname = mndGetDbStr(pVgroup->dbName);
      char        b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
      if (dbname != NULL) {
        STR_WITH_MAXSIZE_TO_VARSTR(b1, dbname, TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE);
      } else {
        STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE);
      }
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
992
      colDataSetVal(pColInfo, numOfRows, (const char *)b1, false);
S
Shengliang Guan 已提交
993 994

      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
995
      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false);
S
Shengliang Guan 已提交
996 997 998 999 1000 1001 1002 1003

      SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
      char       b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
      if (pDnode != NULL) {
        STR_WITH_MAXSIZE_TO_VARSTR(b2, pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE);
      } else {
        STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE);
      }
1004
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
1005
      colDataSetVal(pColInfo, numOfRows, (const char *)b2, false);
H
Haojun Liao 已提交
1006

S
Shengliang Guan 已提交
1007 1008 1009 1010 1011 1012
      numOfRows++;
    }

    sdbRelease(pSdb, pVgroup);
  }

1013
  pShow->numOfRows += numOfRows;
S
Shengliang Guan 已提交
1014 1015 1016 1017 1018 1019
  return numOfRows;
}

static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) {
  SSdb *pSdb = pMnode->pSdb;
  sdbCancelFetch(pSdb, pIter);
L
Liu Jicong 已提交
1020
}
S
Shengliang Guan 已提交
1021

1022
static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray) {
S
Shengliang Guan 已提交
1023 1024 1025
  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
    SDnodeObj *pDnode = taosArrayGet(pArray, i);
1026
    mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes);
S
Shengliang Guan 已提交
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
  }

  SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica];
  for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) {
    SDnodeObj *pDnode = taosArrayGet(pArray, d);

    bool used = false;
    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
      if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) {
        used = true;
        break;
      }
    }
    if (used) continue;

1042
    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
S
Shengliang Guan 已提交
1043 1044 1045 1046
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      return -1;
    }

1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
    if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
             pVgroup->dbName, pVgroup->vgId, vgMem, pDnode->id, pDnode->memAvail, pDnode->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
      return -1;
    } else {
      pDnode->memUsed += vgMem;
    }

S
Shengliang Guan 已提交
1057
    pVgid->dnodeId = pDnode->id;
S
Shengliang Guan 已提交
1058
    pVgid->syncState = TAOS_SYNC_STATE_OFFLINE;
1059 1060
    mInfo("db:%s, vgId:%d, vn:%d is added, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64,
          pVgroup->dbName, pVgroup->vgId, pVgroup->replica, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed);
S
Shengliang Guan 已提交
1061 1062 1063

    pVgroup->replica++;
    pDnode->numOfVnodes++;
1064 1065 1066 1067 1068 1069 1070 1071 1072

    SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
    if (pVgRaw == NULL) return -1;
    if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
      sdbFreeRaw(pVgRaw);
      return -1;
    }
    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);

S
Shengliang Guan 已提交
1073 1074 1075 1076 1077 1078 1079 1080
    return 0;
  }

  terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
  mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
  return -1;
}

1081 1082
static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray,
                                        SVnodeGid *pDelVgid) {
S
Shengliang Guan 已提交
1083 1084 1085
  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
    SDnodeObj *pDnode = taosArrayGet(pArray, i);
1086
    mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes);
S
Shengliang Guan 已提交
1087 1088 1089 1090 1091 1092 1093 1094 1095
  }

  int32_t code = -1;
  for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) {
    SDnodeObj *pDnode = taosArrayGet(pArray, d);

    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
      SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
      if (pVgid->dnodeId == pDnode->id) {
1096 1097 1098 1099
        int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
        pDnode->memUsed -= vgMem;
        mInfo("db:%s, vgId:%d, vn:%d is removed, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64,
              pVgroup->dbName, pVgroup->vgId, vn, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed);
S
Shengliang Guan 已提交
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
        pDnode->numOfVnodes--;
        pVgroup->replica--;
        *pDelVgid = *pVgid;
        *pVgid = pVgroup->vnodeGid[pVgroup->replica];
        memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
        code = 0;
        goto _OVER;
      }
    }
  }

_OVER:
  if (code != 0) {
    terrno = TSDB_CODE_APP_ERROR;
    mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
    return -1;
  }

  for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
    SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
    mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
  }
1122 1123 1124 1125 1126 1127 1128 1129 1130

  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
  if (pVgRaw == NULL) return -1;
  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
    sdbFreeRaw(pVgRaw);
    return -1;
  }
  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);

S
Shengliang Guan 已提交
1131 1132 1133
  return 0;
}

S
Shengliang Guan 已提交
1134
int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) {
S
Shengliang Guan 已提交
1135 1136 1137 1138 1139 1140 1141 1142
  STransAction action = {0};

  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
  if (pDnode == NULL) return -1;
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
S
Shengliang Guan 已提交
1143
  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
S
Shengliang Guan 已提交
1144 1145 1146 1147 1148
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_DND_CREATE_VNODE;
1149
  action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;
S
Shengliang Guan 已提交
1150 1151 1152 1153 1154 1155 1156 1157 1158

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

C
cadem 已提交
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
int32_t mndRestoreAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pDnode) {
  STransAction action = {0};

  action.epSet = mndGetDnodeEpset(pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_DND_CREATE_VNODE;
  action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

S
Shengliang Guan 已提交
1181 1182 1183 1184
int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
  STransAction action = {0};
  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);

1185
  mInfo("vgId:%d, build alter vnode confirm req", pVgroup->vgId);
S
Shengliang Guan 已提交
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
  int32_t   contLen = sizeof(SMsgHead);
  SMsgHead *pHead = taosMemoryMalloc(contLen);
  if (pHead == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return -1;
  }

  pHead->contLen = htonl(contLen);
  pHead->vgId = htonl(pVgroup->vgId);

  action.pCont = pHead;
  action.contLen = contLen;
  action.msgType = TDMT_VND_ALTER_CONFIRM;
1199 1200
  // incorrect redirect result will cause this erro
  action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID;
S
Shengliang Guan 已提交
1201 1202 1203 1204 1205 1206 1207 1208 1209

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pHead);
    return -1;
  }

  return 0;
}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dstVgId) {
  STransAction action = {0};
  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);

  int32_t contLen = 0;
  void   *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, pVgroup, dstVgId, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_VND_ALTER_HASHRANGE;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}
S
Shengliang Guan 已提交
1229 1230

int32_t mndAddAlterVnodeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
S
Shengliang Guan 已提交
1231 1232 1233 1234
  STransAction action = {0};
  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);

  int32_t contLen = 0;
S
Shengliang Guan 已提交
1235
  void   *pReq = mndBuildAlterVnodeConfigReq(pMnode, pDb, pVgroup, &contLen);
S
Shengliang Guan 已提交
1236 1237 1238 1239
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
S
Shengliang Guan 已提交
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
  action.msgType = TDMT_VND_ALTER_CONFIG;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) {
  SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
  if (pDnode == NULL) return -1;

  STransAction action = {0};
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_VND_ALTER_REPLICA;
S
Shengliang Guan 已提交
1265 1266 1267 1268 1269 1270 1271 1272 1273

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

C
cadem 已提交
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) {
  SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
  if (pDnode == NULL) return -1;

  STransAction action = {0};
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_DND_ALTER_VNODE_TYPE;
C
cadem 已提交
1289 1290
  action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER;
  action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP;
C
cadem 已提交
1291 1292 1293 1294 1295 1296 1297 1298 1299

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

H
Haojun Liao 已提交
1300
int32_t mndRestoreAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
C
cadem 已提交
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
                                          SDnodeObj *pDnode) {
  STransAction action = {0};
  action.epSet = mndGetDnodeEpset(pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, pDnode->id, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_DND_ALTER_VNODE_TYPE;
  action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER;
  action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

X
Xiaoyu Wang 已提交
1323 1324
static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
                                             int32_t dnodeId) {
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
  SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
  if (pDnode == NULL) return -1;

  STransAction action = {0};
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildDisableVnodeWriteReq(pMnode, pDb, pVgroup->vgId, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_VND_DISABLE_WRITE;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

S
Shengliang Guan 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
                              bool isRedo) {
  STransAction action = {0};

  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
  if (pDnode == NULL) return -1;
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
  void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_DND_DROP_VNODE;
1364
  action.acceptableCode = TSDB_CODE_VND_NOT_EXIST;
S
Shengliang Guan 已提交
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381

  if (isRedo) {
    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
      taosMemoryFree(pReq);
      return -1;
    }
  } else {
    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
      taosMemoryFree(pReq);
      return -1;
    }
  }

  return 0;
}

int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex,
D
dmchen 已提交
1382
                                    SArray *pArray, bool force, bool unsafe) {
S
Shengliang Guan 已提交
1383 1384
  SVgObj newVg = {0};
  memcpy(&newVg, pVgroup, sizeof(SVgObj));
S
Shengliang Guan 已提交
1385

S
Shengliang Guan 已提交
1386 1387 1388
  mInfo("vgId:%d, vgroup info before move, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
S
Shengliang Guan 已提交
1389 1390
  }

S
Shengliang Guan 已提交
1391
  if (!force) {
1392 1393 1394
#if 1
    {
#else
1395
    if (newVg.replica == 1) {
1396 1397
#endif
      mInfo("vgId:%d, will add 1 vnode, replca:%d", pVgroup->vgId, newVg.replica);
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
      if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
      for (int32_t i = 0; i < newVg.replica - 1; ++i) {
        if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
      }
      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1;
      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;

      mInfo("vgId:%d, will remove 1 vnode, replca:2", pVgroup->vgId);
      newVg.replica--;
      SVnodeGid del = newVg.vnodeGid[vnIndex];
      newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
      memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
      {
        SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
        if (pRaw == NULL) return -1;
        if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
          sdbFreeRaw(pRaw);
          return -1;
        }
        (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
1418 1419
      }

1420 1421 1422 1423 1424
      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
      for (int32_t i = 0; i < newVg.replica; ++i) {
        if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
      }
      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
1425 1426 1427
#if 1
    }
#else
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
    } else {  // new replica == 3
      mInfo("vgId:%d, will add 1 vnode, replca:3", pVgroup->vgId);
      if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
      mInfo("vgId:%d, will remove 1 vnode, replca:4", pVgroup->vgId);
      newVg.replica--;
      SVnodeGid del = newVg.vnodeGid[vnIndex];
      newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
      memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
      {
        SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
        if (pRaw == NULL) return -1;
        if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
          sdbFreeRaw(pRaw);
          return -1;
        }
        (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
      }

      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
1447
      for (int32_t i = 0; i < newVg.replica; ++i) {
1448 1449 1450 1451 1452
        if (i == vnIndex) continue;
        if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
      }
      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1;
      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
S
Shengliang Guan 已提交
1453
    }
1454
#endif
S
Shengliang Guan 已提交
1455 1456
  } else {
    mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId);
1457
    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
S
Shengliang Guan 已提交
1458 1459 1460 1461
    newVg.replica--;
    SVnodeGid del = newVg.vnodeGid[vnIndex];
    newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
    memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
1462 1463 1464 1465 1466 1467 1468 1469 1470
    {
      SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
      if (pRaw == NULL) return -1;
      if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
        sdbFreeRaw(pRaw);
        return -1;
      }
      (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
    }
S
Shengliang Guan 已提交
1471 1472 1473 1474 1475 1476

    for (int32_t i = 0; i < newVg.replica; ++i) {
      if (i != vnIndex) {
        if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
      }
    }
1477
    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1;
S
Shengliang Guan 已提交
1478 1479
    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;

C
cadem 已提交
1480
    if(newVg.replica == 1){
D
dmchen 已提交
1481 1482 1483 1484 1485
      if(force && !unsafe){
        terrno = TSDB_CODE_VND_META_DATA_UNSAFE_DELETE;
        return -1;
      }
      
C
cadem 已提交
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504
      SSdb   *pSdb = pMnode->pSdb;
      void   *pIter = NULL; 

      while (1) {
        SStbObj *pStb = NULL;
        pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb);
        if (pIter == NULL) break;

        if (strcmp(pStb->db, pDb->name) == 0) {
          if (mndSetForceDropCreateStbRedoActions(pMnode, pTrans, &newVg, pStb) != 0) {
            sdbCancelFetch(pSdb, pIter);
            sdbRelease(pSdb, pStb);
            return -1;
          }
        }

        sdbRelease(pSdb, pStb);
      }

S
Shengliang Guan 已提交
1505
      mInfo("vgId:%d, all data is dropped since replica=1", pVgroup->vgId);
S
Shengliang Guan 已提交
1506
    }
S
Shengliang Guan 已提交
1507
  }
S
Shengliang Guan 已提交
1508

1509 1510
  {
    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
1511 1512 1513 1514 1515
    if (pRaw == NULL) return -1;
    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
      sdbFreeRaw(pRaw);
      return -1;
    }
S
Shengliang Guan 已提交
1516
    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
1517
  }
S
Shengliang Guan 已提交
1518

S
Shengliang Guan 已提交
1519 1520 1521
  mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
S
Shengliang Guan 已提交
1522 1523 1524 1525
  }
  return 0;
}

D
dmchen 已提交
1526
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force, bool unsafe) {
S
Shengliang Guan 已提交
1527
  int32_t code = 0;
S
Shengliang Guan 已提交
1528
  SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId);
S
Shengliang Guan 已提交
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
  if (pArray == NULL) return -1;

  void *pIter = NULL;
  while (1) {
    SVgObj *pVgroup = NULL;
    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
    if (pIter == NULL) break;

    int32_t vnIndex = -1;
    for (int32_t i = 0; i < pVgroup->replica; ++i) {
S
Shengliang Guan 已提交
1539
      if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) {
S
Shengliang Guan 已提交
1540 1541 1542 1543 1544
        vnIndex = i;
        break;
      }
    }

S
Shengliang Guan 已提交
1545
    code = 0;
S
Shengliang Guan 已提交
1546
    if (vnIndex != -1) {
S
Shengliang Guan 已提交
1547
      mInfo("vgId:%d, vnode:%d will be removed from dnode:%d, force:%d", pVgroup->vgId, vnIndex, delDnodeId, force);
S
Shengliang Guan 已提交
1548
      SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
D
dmchen 已提交
1549
      code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray, force, unsafe);
S
Shengliang Guan 已提交
1550 1551 1552 1553
      mndReleaseDb(pMnode, pDb);
    }

    sdbRelease(pMnode->pSdb, pVgroup);
S
Shengliang Guan 已提交
1554 1555 1556 1557 1558

    if (code != 0) {
      sdbCancelFetch(pMnode->pSdb, pIter);
      break;
    }
S
Shengliang Guan 已提交
1559 1560 1561
  }

  taosArrayDestroy(pArray);
S
Shengliang Guan 已提交
1562
  return code;
S
Shengliang Guan 已提交
1563 1564 1565 1566
}

static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
                                             int32_t newDnodeId) {
1567
  mInfo("vgId:%d, will add 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId);
S
Shengliang Guan 已提交
1568 1569 1570 1571

  SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica];
  pVgroup->replica++;
  pGid->dnodeId = newDnodeId;
S
Shengliang Guan 已提交
1572
  pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
S
Shengliang Guan 已提交
1573

1574 1575 1576 1577 1578 1579 1580 1581
  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
  if (pVgRaw == NULL) return -1;
  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
    sdbFreeRaw(pVgRaw);
    return -1;
  }
  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);

S
Shengliang Guan 已提交
1582 1583
  for (int32_t i = 0; i < pVgroup->replica - 1; ++i) {
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
S
Shengliang Guan 已提交
1584
  }
1585
  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid) != 0) return -1;
S
Shengliang Guan 已提交
1586
  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
S
Shengliang Guan 已提交
1587 1588 1589 1590 1591 1592

  return 0;
}

static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
                                               int32_t delDnodeId) {
1593
  mInfo("vgId:%d, will remove 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId);
S
Shengliang Guan 已提交
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605

  SVnodeGid *pGid = NULL;
  SVnodeGid  delGid = {0};
  for (int32_t i = 0; i < pVgroup->replica; ++i) {
    if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) {
      pGid = &pVgroup->vnodeGid[i];
      break;
    }
  }

  if (pGid == NULL) return 0;

1606
  pVgroup->replica--;
S
Shengliang Guan 已提交
1607 1608 1609 1610
  memcpy(&delGid, pGid, sizeof(SVnodeGid));
  memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid));
  memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));

1611 1612 1613 1614 1615 1616 1617 1618
  SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
  if (pVgRaw == NULL) return -1;
  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
    sdbFreeRaw(pVgRaw);
    return -1;
  }
  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);

S
Shengliang Guan 已提交
1619 1620 1621
  if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
  for (int32_t i = 0; i < pVgroup->replica; ++i) {
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
S
Shengliang Guan 已提交
1622
  }
S
Shengliang Guan 已提交
1623
  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
S
Shengliang Guan 已提交
1624 1625 1626 1627 1628 1629 1630

  return 0;
}

static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1,
                                     SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3,
                                     SDnodeObj *pOld3) {
1631 1632
  int32_t code = -1;
  STrans *pTrans = NULL;
S
Shengliang Guan 已提交
1633

1634
  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup");
S
Shengliang Guan 已提交
1635 1636
  if (pTrans == NULL) goto _OVER;
  mndTransSetSerial(pTrans);
1637
  mInfo("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId);
S
Shengliang Guan 已提交
1638 1639 1640 1641 1642

  SVgObj newVg = {0};
  memcpy(&newVg, pVgroup, sizeof(SVgObj));
  mInfo("vgId:%d, vgroup info before redistribute, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
S
Shengliang Guan 已提交
1643
    mInfo("vgId:%d, vnode:%d dnode:%d role:%s", newVg.vgId, i, newVg.vnodeGid[i].dnodeId,
1644
          syncStr(newVg.vnodeGid[i].syncState));
S
Shengliang Guan 已提交
1645 1646
  }

S
Shengliang Guan 已提交
1647
  if (pNew1 != NULL && pOld1 != NULL) {
1648 1649 1650 1651 1652 1653 1654
    int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew1->id);
    if (numOfVnodes >= pNew1->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes,
             pNew1->numOfSupportVnodes);
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      goto _OVER;
    }
1655 1656 1657 1658 1659 1660

    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
    if (pNew1->memAvail - vgMem - pNew1->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
             pVgroup->dbName, pVgroup->vgId, vgMem, pNew1->id, pNew1->memAvail, pNew1->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
S
Shengliang Guan 已提交
1661
      goto _OVER;
1662 1663 1664 1665
    } else {
      pNew1->memUsed += vgMem;
    }

1666 1667 1668
    if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER;
    if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER;
  }
S
Shengliang Guan 已提交
1669 1670

  if (pNew2 != NULL && pOld2 != NULL) {
1671 1672 1673 1674 1675 1676 1677
    int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew2->id);
    if (numOfVnodes >= pNew2->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes,
             pNew2->numOfSupportVnodes);
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      goto _OVER;
    }
1678 1679 1680 1681 1682
    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
    if (pNew2->memAvail - vgMem - pNew2->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
             pVgroup->dbName, pVgroup->vgId, vgMem, pNew2->id, pNew2->memAvail, pNew2->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
S
Shengliang Guan 已提交
1683
      goto _OVER;
1684 1685 1686
    } else {
      pNew2->memUsed += vgMem;
    }
S
Shengliang Guan 已提交
1687 1688
    if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER;
    if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER;
1689
  }
S
Shengliang Guan 已提交
1690 1691

  if (pNew3 != NULL && pOld3 != NULL) {
1692 1693 1694 1695 1696 1697 1698
    int32_t numOfVnodes = mndGetVnodesNum(pMnode, pNew3->id);
    if (numOfVnodes >= pNew3->numOfSupportVnodes) {
      mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes,
             pNew3->numOfSupportVnodes);
      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
      goto _OVER;
    }
1699 1700 1701 1702 1703
    int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
    if (pNew3->memAvail - vgMem - pNew3->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64,
             pVgroup->dbName, pVgroup->vgId, vgMem, pNew3->id, pNew3->memAvail, pNew3->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
1704
      goto _OVER;
1705 1706 1707
    } else {
      pNew3->memUsed += vgMem;
    }
S
Shengliang Guan 已提交
1708 1709 1710 1711
    if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id) != 0) goto _OVER;
    if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER;
  }

1712
  {
1713
    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
S
Shengliang Guan 已提交
1714
    if (pRaw == NULL) goto _OVER;
1715 1716
    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
      sdbFreeRaw(pRaw);
S
Shengliang Guan 已提交
1717
      goto _OVER;
1718
    }
S
Shengliang Guan 已提交
1719
    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
1720
  }
S
Shengliang Guan 已提交
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736

  mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
  }

  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
  code = 0;

_OVER:
  mndTransDrop(pTrans);
  mndReleaseDb(pMnode, pDb);
  return code;
}

static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
  SMnode    *pMnode = pReq->info.node;
  SDnodeObj *pNew1 = NULL;
  SDnodeObj *pNew2 = NULL;
  SDnodeObj *pNew3 = NULL;
  SDnodeObj *pOld1 = NULL;
  SDnodeObj *pOld2 = NULL;
  SDnodeObj *pOld3 = NULL;
  SVgObj    *pVgroup = NULL;
  SDbObj    *pDb = NULL;
  int32_t    code = -1;
  int64_t    curMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
1748 1749 1750 1751
  int32_t    newDnodeId[3] = {0};
  int32_t    oldDnodeId[3] = {0};
  int32_t    newIndex = -1;
  int32_t    oldIndex = -1;
1752

S
Shengliang Guan 已提交
1753 1754
  SRedistributeVgroupReq req = {0};
  if (tDeserializeSRedistributeVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) {
S
Shengliang Guan 已提交
1755 1756 1757 1758
    terrno = TSDB_CODE_INVALID_MSG;
    goto _OVER;
  }

S
Shengliang Guan 已提交
1759
  mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3);
1760
  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) {
S
Shengliang Guan 已提交
1761 1762
    goto _OVER;
  }
S
Shengliang Guan 已提交
1763

S
Shengliang Guan 已提交
1764
  pVgroup = mndAcquireVgroup(pMnode, req.vgId);
S
Shengliang Guan 已提交
1765 1766 1767 1768 1769 1770
  if (pVgroup == NULL) goto _OVER;

  pDb = mndAcquireDb(pMnode, pVgroup->dbName);
  if (pDb == NULL) goto _OVER;

  if (pVgroup->replica == 1) {
S
Shengliang Guan 已提交
1771
    if (req.dnodeId1 <= 0 || req.dnodeId2 > 0 || req.dnodeId3 > 0) {
1772 1773 1774
      terrno = TSDB_CODE_MND_INVALID_REPLICA;
      goto _OVER;
    }
S
Shengliang Guan 已提交
1775 1776

    if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) {
1777 1778
      // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
      code = 0;
S
Shengliang Guan 已提交
1779 1780
      goto _OVER;
    }
S
Shengliang Guan 已提交
1781 1782 1783 1784 1785

    pNew1 = mndAcquireDnode(pMnode, req.dnodeId1);
    if (pNew1 == NULL) goto _OVER;
    if (!mndIsDnodeOnline(pNew1, curMs)) {
      terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
S
Shengliang Guan 已提交
1786 1787
      goto _OVER;
    }
S
Shengliang Guan 已提交
1788 1789 1790 1791

    pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId);
    if (pOld1 == NULL) goto _OVER;
    if (!mndIsDnodeOnline(pOld1, curMs)) {
1792 1793 1794
      terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
      goto _OVER;
    }
S
Shengliang Guan 已提交
1795

1796
    code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, NULL, NULL, NULL, NULL);
S
Shengliang Guan 已提交
1797

1798
  } else if (pVgroup->replica == 3) {
S
Shengliang Guan 已提交
1799
    if (req.dnodeId1 <= 0 || req.dnodeId2 <= 0 || req.dnodeId3 <= 0) {
1800 1801 1802
      terrno = TSDB_CODE_MND_INVALID_REPLICA;
      goto _OVER;
    }
S
Shengliang Guan 已提交
1803 1804

    if (req.dnodeId1 == req.dnodeId2 || req.dnodeId1 == req.dnodeId3 || req.dnodeId2 == req.dnodeId3) {
1805 1806 1807
      terrno = TSDB_CODE_MND_INVALID_REPLICA;
      goto _OVER;
    }
S
Shengliang Guan 已提交
1808 1809 1810 1811

    if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId &&
        req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId) {
      newDnodeId[++newIndex] = req.dnodeId1;
S
Shengliang Guan 已提交
1812
      mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex);
S
Shengliang Guan 已提交
1813
    }
S
Shengliang Guan 已提交
1814 1815 1816 1817

    if (req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId &&
        req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId) {
      newDnodeId[++newIndex] = req.dnodeId2;
S
Shengliang Guan 已提交
1818
      mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex);
S
Shengliang Guan 已提交
1819 1820 1821 1822 1823
    }

    if (req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId &&
        req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) {
      newDnodeId[++newIndex] = req.dnodeId3;
S
Shengliang Guan 已提交
1824
      mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex);
S
Shengliang Guan 已提交
1825 1826 1827 1828 1829
    }

    if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId &&
        req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId) {
      oldDnodeId[++oldIndex] = pVgroup->vnodeGid[0].dnodeId;
S
Shengliang Guan 已提交
1830
      mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex);
S
Shengliang Guan 已提交
1831 1832 1833 1834 1835
    }

    if (req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId &&
        req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId) {
      oldDnodeId[++oldIndex] = pVgroup->vnodeGid[1].dnodeId;
S
Shengliang Guan 已提交
1836
      mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex);
S
Shengliang Guan 已提交
1837 1838 1839 1840 1841
    }

    if (req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId &&
        req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) {
      oldDnodeId[++oldIndex] = pVgroup->vnodeGid[2].dnodeId;
S
Shengliang Guan 已提交
1842
      mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex);
S
Shengliang Guan 已提交
1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
    }

    if (newDnodeId[0] != 0) {
      pNew1 = mndAcquireDnode(pMnode, newDnodeId[0]);
      if (pNew1 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pNew1, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (newDnodeId[1] != 0) {
      pNew2 = mndAcquireDnode(pMnode, newDnodeId[1]);
      if (pNew2 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pNew2, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (newDnodeId[2] != 0) {
      pNew3 = mndAcquireDnode(pMnode, newDnodeId[2]);
      if (pNew3 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pNew3, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (oldDnodeId[0] != 0) {
      pOld1 = mndAcquireDnode(pMnode, oldDnodeId[0]);
      if (pOld1 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pOld1, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (oldDnodeId[1] != 0) {
      pOld2 = mndAcquireDnode(pMnode, oldDnodeId[1]);
      if (pOld2 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pOld2, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (oldDnodeId[2] != 0) {
      pOld3 = mndAcquireDnode(pMnode, oldDnodeId[2]);
      if (pOld3 == NULL) goto _OVER;
      if (!mndIsDnodeOnline(pOld3, curMs)) {
        terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
        goto _OVER;
      }
    }

    if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) {
1900 1901
      // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
      code = 0;
1902 1903
      goto _OVER;
    }
S
Shengliang Guan 已提交
1904

1905
    code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3);
S
Shengliang Guan 已提交
1906

1907 1908 1909
  } else {
    terrno = TSDB_CODE_MND_INVALID_REPLICA;
    goto _OVER;
S
Shengliang Guan 已提交
1910 1911 1912 1913 1914 1915
  }

  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;

_OVER:
  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
S
Shengliang Guan 已提交
1916 1917
    mError("vgId:%d, failed to redistribute to dnode %d:%d:%d since %s", req.vgId, req.dnodeId1, req.dnodeId2,
           req.dnodeId3, terrstr());
S
Shengliang Guan 已提交
1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
  }

  mndReleaseDnode(pMnode, pNew1);
  mndReleaseDnode(pMnode, pNew2);
  mndReleaseDnode(pMnode, pNew3);
  mndReleaseDnode(pMnode, pOld1);
  mndReleaseDnode(pMnode, pOld2);
  mndReleaseDnode(pMnode, pOld3);
  mndReleaseVgroup(pMnode, pVgroup);
  mndReleaseDb(pMnode, pDb);

  return code;
}

C
cadem 已提交
1932
static void *mndBuildSForceBecomeFollowerReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dnodeId,
C
cadem 已提交
1933
                                          int32_t *pContLen) {
C
cadem 已提交
1934
  SForceBecomeFollowerReq balanceReq = {
C
cadem 已提交
1935 1936 1937
      .vgId = pVgroup->vgId,
  };

C
cadem 已提交
1938
  int32_t contLen = tSerializeSForceBecomeFollowerReq(NULL, 0, &balanceReq);
C
cadem 已提交
1939 1940 1941 1942
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }
C
cadem 已提交
1943
  contLen += sizeof(SMsgHead);
C
cadem 已提交
1944 1945 1946 1947 1948 1949 1950

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

C
cadem 已提交
1951 1952 1953 1954 1955
  SMsgHead *pHead = pReq;
  pHead->contLen = htonl(contLen);
  pHead->vgId = htonl(pVgroup->vgId);

  tSerializeSForceBecomeFollowerReq((char *)pReq + sizeof(SMsgHead), contLen, &balanceReq);
C
cadem 已提交
1956
  *pContLen = contLen;
C
cadem 已提交
1957
  return pReq;                                  
C
cadem 已提交
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
}

int32_t mndAddBalanceVgroupLeaderAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dnodeId) {
  SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
  if (pDnode == NULL) return -1;

  STransAction action = {0};
  action.epSet = mndGetDnodeEpset(pDnode);
  mndReleaseDnode(pMnode, pDnode);

  int32_t contLen = 0;
C
cadem 已提交
1969
  void   *pReq = mndBuildSForceBecomeFollowerReq(pMnode, pVgroup, dnodeId, &contLen);
C
cadem 已提交
1970 1971 1972 1973
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
C
cadem 已提交
1974
  action.msgType = TDMT_SYNC_FORCE_FOLLOWER;
C
cadem 已提交
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTrans){
  SSdb *pSdb = pMnode->pSdb;

  int32_t vgid = pVgroup->vgId;
  int8_t replica = pVgroup->replica;

D
dmchen 已提交
1990
  if(pVgroup->replica <= 1) {
C
cadem 已提交
1991 1992 1993 1994
    mInfo("trans:%d, vgid:%d no need to balance, replica:%d", pTrans->id, vgid, replica);
    return -1;
  }

C
cadem 已提交
1995 1996 1997 1998 1999 2000 2001 2002 2003
  int32_t dnodeId = pVgroup->vnodeGid[0].dnodeId;

  for(int i = 0; i < replica; i++)
  {
    if(pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER){
      dnodeId = pVgroup->vnodeGid[i].dnodeId;
      break;
    }
  }
C
cadem 已提交
2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023

  bool       exist = false;
  bool       online = false;
  int64_t curMs = taosGetTimestampMs();
  SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
  if (pDnode != NULL) {
    exist = true;
    online = mndIsDnodeOnline(pDnode, curMs);
    mndReleaseDnode(pMnode, pDnode);
  }

  if(exist && online)
  {
    mInfo("trans:%d, vgid:%d leader to dnode:%d", pTrans->id, vgid, dnodeId);

    if (mndAddBalanceVgroupLeaderAction(pMnode, pTrans, pVgroup, dnodeId) != 0) {
      mError("trans:%d, vgid:%d failed to be balanced to dnode:%d", pTrans->id, vgid, dnodeId);
      return -1;
    }

D
dmchen 已提交
2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
    SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
    if (pDb == NULL) {
      mError("trans:%d, vgid:%d failed to be balanced to dnode:%d, because db not exist", pTrans->id, vgid, dnodeId);
      return -1;
    }

    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) {
      mError("trans:%d, vgid:%d failed to be balanced to dnode:%d", pTrans->id, vgid, dnodeId);
      return -1;
    }

    mndReleaseDb(pMnode, pDb);

C
cadem 已提交
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
    SSdbRaw *pRaw = mndVgroupActionEncode(pVgroup);
    if (pRaw == NULL) {
      mError("trans:%d, vgid:%d failed to encode action to dnode:%d", pTrans->id, vgid, dnodeId);
      return -1;
    }
    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
      sdbFreeRaw(pRaw);
      mError("trans:%d, vgid:%d failed to append commit log dnode:%d", pTrans->id, vgid, dnodeId);
      return -1;
    }
    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
  }
  else
  {
H
Haojun Liao 已提交
2051
    mInfo("trans:%d, vgid:%d cant be balanced to dnode:%d, exist:%d, online:%d",
D
dmchen 已提交
2052
                              pTrans->id, vgid, dnodeId, exist, online);
C
cadem 已提交
2053 2054 2055 2056 2057
  }

  return 0;
}

C
cadem 已提交
2058
extern int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq);
C
cadem 已提交
2059

C
cadem 已提交
2060 2061 2062
int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) {
  return mndProcessVgroupBalanceLeaderMsgImp(pReq);
}
C
cadem 已提交
2063

C
cadem 已提交
2064 2065 2066
#ifndef TD_ENTERPRISE
int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq) {
  return 0;
C
cadem 已提交
2067
}
C
cadem 已提交
2068
#endif
C
cadem 已提交
2069

2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094
static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup,
                                   SVgObj *pNewVgroup, SArray *pArray) {
  for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) {
    SDnodeObj *pDnode = taosArrayGet(pArray, i);
    bool       inVgroup = false;
    for (int32_t j = 0; j < pOldVgroup->replica; ++j) {
      SVnodeGid *pVgId = &pOldVgroup->vnodeGid[i];
      if (pDnode->id == pVgId->dnodeId) {
        pDnode->memUsed -= mndGetVgroupMemory(pMnode, pOldDb, pOldVgroup);
        inVgroup = true;
      }
    }
    for (int32_t j = 0; j < pNewVgroup->replica; ++j) {
      SVnodeGid *pVgId = &pNewVgroup->vnodeGid[i];
      if (pDnode->id == pVgId->dnodeId) {
        pDnode->memUsed += mndGetVgroupMemory(pMnode, pNewDb, pNewVgroup);
        inVgroup = true;
      }
    }
    if (pDnode->memAvail - pDnode->memUsed <= 0) {
      mError("db:%s, vgId:%d, no enough memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName,
             pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed);
      terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE;
      return -1;
    } else if (inVgroup) {
H
Hongze Cheng 已提交
2095 2096
      mInfo("db:%s, vgId:%d, memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName, pNewVgroup->vgId,
            pDnode->id, pDnode->memAvail, pDnode->memUsed);
2097 2098
    } else {
    }
S
Shengliang Guan 已提交
2099
  }
2100 2101
  return 0;
}
S
Shengliang Guan 已提交
2102

2103 2104
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
                                  SArray *pArray) {
S
Shengliang Guan 已提交
2105 2106
  SVgObj newVgroup = {0};
  memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
2107 2108

  if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) {
S
Shengliang Guan 已提交
2109
    if (mndAddAlterVnodeConfigAction(pMnode, pTrans, pNewDb, pVgroup) != 0) return -1;
2110 2111 2112 2113
    if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1;
    return 0;
  }

S
Shengliang Guan 已提交
2114 2115
  mndTransSetSerial(pTrans);

S
Shengliang Guan 已提交
2116 2117
  if (newVgroup.replica == 1 && pNewDb->cfg.replications == 3) {
    mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
S
Shengliang Guan 已提交
2118 2119
          pVgroup->vnodeGid[0].dnodeId);

C
cadem 已提交
2120
    //add first
2121
    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
C
cadem 已提交
2122 2123 2124

    newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
    newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER;
S
Shengliang Guan 已提交
2125 2126
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
      return -1;
C
cadem 已提交
2127

2128
    if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
C
cadem 已提交
2129 2130 2131 2132
    newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
    if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
      return -1;

2133
    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
2134

C
cadem 已提交
2135
    //add second
2136
    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
C
cadem 已提交
2137 2138 2139 2140
    newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
    newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
    newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER;

2141 2142 2143 2144
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
      return -1;
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
      return -1;
S
Shengliang Guan 已提交
2145
    if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1;
C
cadem 已提交
2146

S
Shengliang Guan 已提交
2147
    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
S
Shengliang Guan 已提交
2148
  } else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) {
S
Shengliang Guan 已提交
2149 2150
    mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName,
          pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId);
S
Shengliang Guan 已提交
2151

S
Shengliang Guan 已提交
2152
    SVnodeGid del1 = {0};
S
Shengliang Guan 已提交
2153
    SVnodeGid del2 = {0};
2154
    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1;
S
Shengliang Guan 已提交
2155
    if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
2156 2157 2158 2159
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
      return -1;
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
      return -1;
S
Shengliang Guan 已提交
2160
    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
2161

2162
    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1;
2163
    if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
S
Shengliang Guan 已提交
2164 2165
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
      return -1;
2166
    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
S
Shengliang Guan 已提交
2167
  } else {
S
Shengliang Guan 已提交
2168
    return -1;
S
Shengliang Guan 已提交
2169
  }
S
Shengliang Guan 已提交
2170

2171 2172
  mndSortVnodeGid(&newVgroup);

S
Shengliang Guan 已提交
2173 2174 2175 2176 2177 2178
  {
    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
    if (pVgRaw == NULL) return -1;
    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
      sdbFreeRaw(pVgRaw);
      return -1;
S
Shengliang Guan 已提交
2179
    }
S
Shengliang Guan 已提交
2180
    (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
S
Shengliang Guan 已提交
2181 2182 2183 2184 2185
  }

  return 0;
}

H
Haojun Liao 已提交
2186
int32_t mndBuildRestoreAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *db, SVgObj *pVgroup,
C
cadem 已提交
2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
                                        SDnodeObj *pDnode) {
  SVgObj newVgroup = {0};
  memcpy(&newVgroup, pVgroup, sizeof(SVgObj));

  mInfo("db:%s, vgId:%d, restore vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
        pVgroup->vnodeGid[0].dnodeId);

  if(newVgroup.replica == 1){
    int selected = 0;
    for(int i = 0; i < newVgroup.replica; i++){
H
Haojun Liao 已提交
2197
      newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER;
C
cadem 已提交
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
      if(newVgroup.vnodeGid[i].dnodeId == pDnode->id){
        selected = i;
      }
    }
    if (mndAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, &newVgroup.vnodeGid[selected]) != 0) return -1;
  }
  else if(newVgroup.replica == 3){
    for(int i = 0; i < newVgroup.replica; i++){
      if(newVgroup.vnodeGid[i].dnodeId == pDnode->id){
        newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_LEARNER;
      }
      else{
        newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER;
H
Haojun Liao 已提交
2211
      }
C
cadem 已提交
2212 2213 2214 2215
    }
    if (mndRestoreAddCreateVnodeAction(pMnode, pTrans, db, &newVgroup, pDnode) != 0) return -1;

    for(int i = 0; i < newVgroup.replica; i++){
H
Haojun Liao 已提交
2216
      newVgroup.vnodeGid[i].nodeRole = TAOS_SYNC_ROLE_VOTER;
C
cadem 已提交
2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234
    if(newVgroup.vnodeGid[i].dnodeId == pDnode->id){
      }
    }
    if (mndRestoreAddAlterVnodeTypeAction(pMnode, pTrans, db, &newVgroup, pDnode) != 0)
        return -1;
  }

  SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
  if (pVgRaw == NULL) return -1;
  if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
    sdbFreeRaw(pVgRaw);
    return -1;
  }
  (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);

  return 0;
}

S
Shengliang Guan 已提交
2235 2236 2237 2238
static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
  return 0;
}

B
Benguang Zhao 已提交
2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250
static int32_t mndTransCommitVgStatus(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus) {
  SSdbRaw *pRaw = mndVgroupActionEncode(pVg);
  if (pRaw == NULL) goto _err;
  if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _err;
  (void)sdbSetRawStatus(pRaw, vgStatus);
  pRaw = NULL;
  return 0;
_err:
  sdbFreeRaw(pRaw);
  return -1;
}

2251
int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
2252 2253 2254
  int32_t  code = -1;
  STrans  *pTrans = NULL;
  SSdbRaw *pRaw = NULL;
2255
  SDbObj   dbObj = {0};
2256
  SArray  *pArray = mndBuildDnodesArray(pMnode, 0);
S
Shengliang Guan 已提交
2257

2258
  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup");
S
Shengliang Guan 已提交
2259 2260
  if (pTrans == NULL) goto _OVER;
  mndTransSetSerial(pTrans);
2261
  mInfo("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId);
S
Shengliang Guan 已提交
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271

  SVgObj newVg1 = {0};
  memcpy(&newVg1, pVgroup, sizeof(SVgObj));
  mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica,
        newVg1.hashBegin, newVg1.hashEnd);
  for (int32_t i = 0; i < newVg1.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
  }

  if (newVg1.replica == 1) {
2272
    if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER;
S
Shengliang Guan 已提交
2273
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
2274
    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER;
S
Shengliang Guan 已提交
2275 2276
  } else if (newVg1.replica == 3) {
    SVnodeGid del1 = {0};
2277
    if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER;
2278
    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
S
Shengliang Guan 已提交
2279 2280
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
    if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER;
S
Shengliang Guan 已提交
2281 2282 2283 2284
  } else {
    goto _OVER;
  }

2285 2286 2287 2288 2289
  for (int32_t i = 0; i < newVg1.replica; ++i) {
    if (mndAddDisableVnodeWriteAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[i].dnodeId) != 0) goto _OVER;
  }
  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;

S
Shengliang Guan 已提交
2290
  SVgObj newVg2 = {0};
S
Shengliang Guan 已提交
2291
  memcpy(&newVg2, &newVg1, sizeof(SVgObj));
S
Shengliang Guan 已提交
2292
  newVg1.replica = 1;
S
Shengliang Guan 已提交
2293
  newVg1.hashEnd = newVg1.hashBegin / 2 + newVg1.hashEnd / 2;
S
Shengliang Guan 已提交
2294 2295 2296 2297 2298
  memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));

  newVg2.replica = 1;
  newVg2.hashBegin = newVg1.hashEnd + 1;
  memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid));
S
Shengliang Guan 已提交
2299 2300
  memset(&newVg2.vnodeGid[1], 0, sizeof(SVnodeGid));

2301 2302
  mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg1.vgId, newVg1.replica,
        newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
2303 2304 2305
  for (int32_t i = 0; i < newVg1.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
  }
2306 2307
  mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg2.vgId, newVg2.replica,
        newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
2308 2309 2310
  for (int32_t i = 0; i < newVg1.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId);
  }
S
Shengliang Guan 已提交
2311

2312
  // alter vgId and hash range
2313 2314 2315 2316 2317 2318 2319
  int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
  if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg1, maxVgId) != 0) goto _OVER;
  newVg1.vgId = maxVgId;

  maxVgId++;
  if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg2, maxVgId) != 0) goto _OVER;
  newVg2.vgId = maxVgId;
S
Shengliang Guan 已提交
2320

B
Benguang Zhao 已提交
2321 2322 2323 2324
  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;

  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER;

S
Shengliang Guan 已提交
2325 2326 2327
  // adjust vgroup replica
  if (pDb->cfg.replications != newVg1.replica) {
    if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER;
2328
  } else {
B
Benguang Zhao 已提交
2329
    if (mndTransCommitVgStatus(pTrans, &newVg1, SDB_STATUS_READY) < 0) goto _OVER;
S
Shengliang Guan 已提交
2330
  }
2331

S
Shengliang Guan 已提交
2332 2333
  if (pDb->cfg.replications != newVg2.replica) {
    if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER;
2334
  } else {
B
Benguang Zhao 已提交
2335
    if (mndTransCommitVgStatus(pTrans, &newVg2, SDB_STATUS_READY) < 0) goto _OVER;
S
Shengliang Guan 已提交
2336 2337
  }

B
Benguang Zhao 已提交
2338
  if (mndTransCommitVgStatus(pTrans, pVgroup, SDB_STATUS_DROPPED) < 0) goto _OVER;
2339

2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352
  memcpy(&dbObj, pDb, sizeof(SDbObj));
  if (dbObj.cfg.pRetensions != NULL) {
    dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
    if (dbObj.cfg.pRetensions == NULL) goto _OVER;
  }
  dbObj.vgVersion++;
  dbObj.updateTime = taosGetTimestampMs();
  dbObj.cfg.numOfVgroups++;
  pRaw = mndDbActionEncode(&dbObj);
  if (pRaw == NULL) goto _OVER;
  if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
  (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
  pRaw = NULL;
S
Shengliang Guan 已提交
2353 2354 2355

  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
  code = 0;
S
Shengliang Guan 已提交
2356 2357

_OVER:
S
Shengliang Guan 已提交
2358
  taosArrayDestroy(pArray);
S
Shengliang Guan 已提交
2359
  mndTransDrop(pTrans);
2360
  sdbFreeRaw(pRaw);
2361
  taosArrayDestroy(dbObj.cfg.pRetensions);
S
Shengliang Guan 已提交
2362 2363 2364
  return code;
}

2365
extern int32_t mndProcessSplitVgroupMsgImp(SRpcMsg *pReq);
S
Shengliang Guan 已提交
2366

2367
static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return mndProcessSplitVgroupMsgImp(pReq); }
S
Shengliang Guan 已提交
2368

2369 2370 2371
#ifndef TD_ENTERPRISE
int32_t mndProcessSplitVgroupMsgImp(SRpcMsg *pReq) { return 0; }
#endif
S
Shengliang Guan 已提交
2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384

static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
                                              SDnodeObj *pSrc, SDnodeObj *pDst) {
  SVgObj newVg = {0};
  memcpy(&newVg, pVgroup, sizeof(SVgObj));
  mInfo("vgId:%d, vgroup info before balance, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
  }

  if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1;
  if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1;

2385 2386
  {
    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
2387 2388
    if (pRaw == NULL) return -1;
    if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
2389 2390 2391
      sdbFreeRaw(pRaw);
      return -1;
    }
S
Shengliang Guan 已提交
2392
    (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
S
Shengliang Guan 已提交
2393 2394 2395 2396 2397 2398 2399 2400 2401
  }

  mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica);
  for (int32_t i = 0; i < newVg.replica; ++i) {
    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
  }
  return 0;
}

S
Shengliang Guan 已提交
2402 2403
static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst,
                                            SHashObj *pBalancedVgroups) {
S
Shengliang Guan 已提交
2404 2405
  void   *pIter = NULL;
  int32_t code = -1;
S
Shengliang Guan 已提交
2406
  SSdb   *pSdb = pMnode->pSdb;
S
Shengliang Guan 已提交
2407 2408 2409

  while (1) {
    SVgObj *pVgroup = NULL;
S
Shengliang Guan 已提交
2410
    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
S
Shengliang Guan 已提交
2411
    if (pIter == NULL) break;
S
Shengliang Guan 已提交
2412 2413 2414 2415
    if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) {
      sdbRelease(pSdb, pVgroup);
      continue;
    }
S
Shengliang Guan 已提交
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425

    bool existInSrc = false;
    bool existInDst = false;
    for (int32_t i = 0; i < pVgroup->replica; ++i) {
      SVnodeGid *pGid = &pVgroup->vnodeGid[i];
      if (pGid->dnodeId == pSrc->id) existInSrc = true;
      if (pGid->dnodeId == pDst->id) existInDst = true;
    }

    if (!existInSrc || existInDst) {
S
Shengliang Guan 已提交
2426 2427
      sdbRelease(pSdb, pVgroup);
      continue;
S
Shengliang Guan 已提交
2428 2429 2430 2431
    }

    SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
    code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
S
Shengliang Guan 已提交
2432 2433 2434
    if (code == 0) {
      code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t));
    }
S
Shengliang Guan 已提交
2435
    mndReleaseDb(pMnode, pDb);
S
Shengliang Guan 已提交
2436 2437
    sdbRelease(pSdb, pVgroup);
    sdbCancelFetch(pSdb, pIter);
S
Shengliang Guan 已提交
2438 2439 2440 2441 2442 2443 2444
    break;
  }

  return code;
}

static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
2445 2446 2447
  int32_t   code = -1;
  int32_t   numOfVgroups = 0;
  STrans   *pTrans = NULL;
S
Shengliang Guan 已提交
2448 2449 2450 2451
  SHashObj *pBalancedVgroups = NULL;

  pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
  if (pBalancedVgroups == NULL) goto _OVER;
S
Shengliang Guan 已提交
2452

2453
  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "balance-vgroup");
S
Shengliang Guan 已提交
2454 2455
  if (pTrans == NULL) goto _OVER;
  mndTransSetSerial(pTrans);
2456
  mInfo("trans:%d, used to balance vgroup", pTrans->id);
S
Shengliang Guan 已提交
2457 2458 2459

  while (1) {
    taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
S
Shengliang Guan 已提交
2460 2461
    for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
      SDnodeObj *pDnode = taosArrayGet(pArray, i);
2462 2463
      mInfo("dnode:%d, equivalent vnodes:%d others:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
            pDnode->numOfSupportVnodes, pDnode->numOfOtherNodes, mndGetDnodeScore(pDnode, 0, 1));
S
Shengliang Guan 已提交
2464 2465 2466 2467
    }

    SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
    SDnodeObj *pDst = taosArrayGet(pArray, 0);
S
Shengliang Guan 已提交
2468

2469 2470 2471
    float srcScore = mndGetDnodeScore(pSrc, -1, 1);
    float dstScore = mndGetDnodeScore(pDst, 1, 1);
    mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, dstScore,
2472
          pDst->id, dstScore);
S
Shengliang Guan 已提交
2473 2474

    if (srcScore > dstScore - 0.000001) {
S
Shengliang Guan 已提交
2475
      code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups);
S
Shengliang Guan 已提交
2476
      if (code == 0) {
S
Shengliang Guan 已提交
2477 2478
        pSrc->numOfVnodes--;
        pDst->numOfVnodes++;
S
Shengliang Guan 已提交
2479 2480 2481
        numOfVgroups++;
        continue;
      } else {
2482
        mInfo("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
S
Shengliang Guan 已提交
2483
        break;
S
Shengliang Guan 已提交
2484 2485
      }
    } else {
2486
      mInfo("trans:%d, no vgroup need to balance any more", pTrans->id);
S
Shengliang Guan 已提交
2487 2488 2489 2490 2491
      break;
    }
  }

  if (numOfVgroups <= 0) {
2492
    mInfo("no need to balance vgroup");
S
Shengliang Guan 已提交
2493 2494
    code = 0;
  } else {
2495
    mInfo("start to balance vgroup, numOfVgroups:%d", numOfVgroups);
S
Shengliang Guan 已提交
2496 2497 2498 2499 2500
    if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
    code = TSDB_CODE_ACTION_IN_PROGRESS;
  }

_OVER:
2501
  taosHashCleanup(pBalancedVgroups);
S
Shengliang Guan 已提交
2502 2503 2504 2505 2506
  mndTransDrop(pTrans);
  return code;
}

static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
2507 2508 2509
  SMnode *pMnode = pReq->info.node;
  int32_t code = -1;
  SArray *pArray = NULL;
S
Shengliang Guan 已提交
2510
  void   *pIter = NULL;
2511
  int64_t curMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
2512

S
Shengliang Guan 已提交
2513 2514 2515 2516 2517 2518 2519
  SBalanceVgroupReq req = {0};
  if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) {
    terrno = TSDB_CODE_INVALID_MSG;
    goto _OVER;
  }

  mInfo("start to balance vgroup");
2520
  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) {
S
Shengliang Guan 已提交
2521 2522
    goto _OVER;
  }
S
Shengliang Guan 已提交
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541

  while (1) {
    SDnodeObj *pDnode = NULL;
    pIter = sdbFetch(pMnode->pSdb, SDB_DNODE, pIter, (void **)&pDnode);
    if (pIter == NULL) break;
    if (!mndIsDnodeOnline(pDnode, curMs)) {
      terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
      mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id);
      sdbRelease(pMnode->pSdb, pDnode);
      goto _OVER;
    }

    sdbRelease(pMnode->pSdb, pDnode);
  }

  pArray = mndBuildDnodesArray(pMnode, 0);
  if (pArray == NULL) goto _OVER;

  if (taosArrayGetSize(pArray) < 2) {
2542
    mInfo("no need to balance vgroup since dnode num less than 2");
S
Shengliang Guan 已提交
2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554
    code = 0;
  } else {
    code = mndBalanceVgroup(pMnode, pReq, pArray);
  }

_OVER:
  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
    mError("failed to balance vgroup since %s", terrstr());
  }

  taosArrayDestroy(pArray);
  return code;
S
Shengliang Guan 已提交
2555 2556
}

L
Liu Jicong 已提交
2557
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }
2558

H
Haojun Liao 已提交
2559
bool mndVgroupInDnode(SVgObj *pVgroup, int32_t dnodeId) {
C
cadem 已提交
2560 2561 2562
  for(int i = 0; i < pVgroup->replica; i++){
    if(pVgroup->vnodeGid[i].dnodeId == dnodeId) return true;
  }
H
Haojun Liao 已提交
2563
  return false;
C
cadem 已提交
2564 2565
}

H
Hongze Cheng 已提交
2566 2567
static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, int64_t compactTs,
                                     STimeWindow tw) {
2568 2569 2570
  SCompactVnodeReq compactReq = {0};
  compactReq.dbUid = pDb->uid;
  compactReq.compactStartTime = compactTs;
H
Hongze Cheng 已提交
2571
  compactReq.tw = tw;
2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
  tstrncpy(compactReq.db, pDb->name, TSDB_DB_FNAME_LEN);

  mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
  int32_t contLen = tSerializeSCompactVnodeReq(NULL, 0, &compactReq);
  if (contLen < 0) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }
  contLen += sizeof(SMsgHead);

  void *pReq = taosMemoryMalloc(contLen);
  if (pReq == NULL) {
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return NULL;
  }

  SMsgHead *pHead = pReq;
  pHead->contLen = htonl(contLen);
  pHead->vgId = htonl(pVgroup->vgId);

  tSerializeSCompactVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &compactReq);
  *pContLen = contLen;
  return pReq;
}

H
Hongze Cheng 已提交
2597 2598
static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
                                        STimeWindow tw) {
2599 2600 2601 2602
  STransAction action = {0};
  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);

  int32_t contLen = 0;
H
Hongze Cheng 已提交
2603
  void   *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs, tw);
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
  if (pReq == NULL) return -1;

  action.pCont = pReq;
  action.contLen = contLen;
  action.msgType = TDMT_VND_COMPACT;

  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
    taosMemoryFree(pReq);
    return -1;
  }

  return 0;
}

H
Hongze Cheng 已提交
2618 2619 2620
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs,
                                    STimeWindow tw) {
  if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs, tw) != 0) return -1;
2621
  return 0;
2622
}