tqRead.c 45.1 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

H
Hongze Cheng 已提交
16
#include "tq.h"
L
Liu Jicong 已提交
17

L
Liu Jicong 已提交
18 19
bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__TABLE) {
20 21 22
    return true;
  }

L
Liu Jicong 已提交
23 24 25
  int16_t msgType = pHead->msgType;
  char*   body = pHead->body;
  int32_t bodyLen = pHead->bodyLen;
26

L
Liu Jicong 已提交
27 28 29 30 31
  int64_t  tbSuid = pHandle->execHandle.execTb.suid;
  int64_t  realTbSuid = 0;
  SDecoder coder;
  void*    data = POINTER_SHIFT(body, sizeof(SMsgHead));
  int32_t  len = bodyLen - sizeof(SMsgHead);
32 33 34 35 36 37 38 39 40 41 42 43 44
  tDecoderInit(&coder, data, len);

  if (msgType == TDMT_VND_CREATE_STB || msgType == TDMT_VND_ALTER_STB) {
    SVCreateStbReq req = {0};
    if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
      goto end;
    }
    realTbSuid = req.suid;
  } else if (msgType == TDMT_VND_DROP_STB) {
    SVDropStbReq req = {0};
    if (tDecodeSVDropStbReq(&coder, &req) < 0) {
      goto end;
    }
L
Liu Jicong 已提交
45
    realTbSuid = req.suid;
46 47 48 49 50 51
  } else if (msgType == TDMT_VND_CREATE_TABLE) {
    SVCreateTbBatchReq req = {0};
    if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) {
      goto end;
    }

L
Liu Jicong 已提交
52
    int32_t        needRebuild = 0;
53 54 55
    SVCreateTbReq* pCreateReq = NULL;
    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
      pCreateReq = req.pReqs + iReq;
L
Liu Jicong 已提交
56
      if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) {
57 58 59
        needRebuild++;
      }
    }
L
Liu Jicong 已提交
60
    if (needRebuild == 0) {
61
      // do nothing
L
Liu Jicong 已提交
62
    } else if (needRebuild == req.nReqs) {
63
      realTbSuid = tbSuid;
L
Liu Jicong 已提交
64
    } else {
65 66 67 68 69
      realTbSuid = tbSuid;
      SVCreateTbBatchReq reqNew = {0};
      reqNew.pArray = taosArrayInit(req.nReqs, sizeof(struct SVCreateTbReq));
      for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
        pCreateReq = req.pReqs + iReq;
L
Liu Jicong 已提交
70
        if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) {
71 72 73 74 75
          reqNew.nReqs++;
          taosArrayPush(reqNew.pArray, pCreateReq);
        }
      }

L
Liu Jicong 已提交
76
      int     tlen;
77 78 79 80 81
      int32_t ret = 0;
      tEncodeSize(tEncodeSVCreateTbBatchReq, &reqNew, tlen, ret);
      void* buf = taosMemoryMalloc(tlen);
      if (NULL == buf) {
        taosArrayDestroy(reqNew.pArray);
82 83 84 85 86 87 88
        for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
          pCreateReq = req.pReqs + iReq;
          taosMemoryFreeClear(pCreateReq->comment);
          if (pCreateReq->type == TSDB_CHILD_TABLE) {
            taosArrayDestroy(pCreateReq->ctb.tagName);
          }
        }
89 90 91 92 93 94 95 96 97 98 99
        goto end;
      }
      SEncoder coderNew = {0};
      tEncoderInit(&coderNew, buf, tlen - sizeof(SMsgHead));
      tEncodeSVCreateTbBatchReq(&coderNew, &reqNew);
      tEncoderClear(&coderNew);
      memcpy(pHead->body + sizeof(SMsgHead), buf, tlen);
      pHead->bodyLen = tlen + sizeof(SMsgHead);
      taosMemoryFree(buf);
      taosArrayDestroy(reqNew.pArray);
    }
100 101 102 103 104 105 106 107

    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
      pCreateReq = req.pReqs + iReq;
      taosMemoryFreeClear(pCreateReq->comment);
      if (pCreateReq->type == TSDB_CHILD_TABLE) {
        taosArrayDestroy(pCreateReq->ctb.tagName);
      }
    }
108
  } else if (msgType == TDMT_VND_ALTER_TABLE) {
L
Liu Jicong 已提交
109
    SVAlterTbReq req = {0};
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

    if (tDecodeSVAlterTbReq(&coder, &req) < 0) {
      goto end;
    }

    SMetaReader mr = {0};
    metaReaderInit(&mr, pHandle->execHandle.pExecReader->pVnodeMeta, 0);

    if (metaGetTableEntryByName(&mr, req.tbName) < 0) {
      metaReaderClear(&mr);
      goto end;
    }
    realTbSuid = mr.me.ctbEntry.suid;
    metaReaderClear(&mr);
  } else if (msgType == TDMT_VND_DROP_TABLE) {
    SVDropTbBatchReq req = {0};

    if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) {
      goto end;
    }

L
Liu Jicong 已提交
131
    int32_t      needRebuild = 0;
132 133 134 135
    SVDropTbReq* pDropReq = NULL;
    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
      pDropReq = req.pReqs + iReq;

L
Liu Jicong 已提交
136
      if (pDropReq->suid == tbSuid) {
137 138 139
        needRebuild++;
      }
    }
L
Liu Jicong 已提交
140
    if (needRebuild == 0) {
141
      // do nothing
L
Liu Jicong 已提交
142
    } else if (needRebuild == req.nReqs) {
143
      realTbSuid = tbSuid;
L
Liu Jicong 已提交
144
    } else {
145 146 147 148 149
      realTbSuid = tbSuid;
      SVDropTbBatchReq reqNew = {0};
      reqNew.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbReq));
      for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
        pDropReq = req.pReqs + iReq;
L
Liu Jicong 已提交
150
        if (pDropReq->suid == tbSuid) {
151 152 153 154 155
          reqNew.nReqs++;
          taosArrayPush(reqNew.pArray, pDropReq);
        }
      }

L
Liu Jicong 已提交
156
      int     tlen;
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
      int32_t ret = 0;
      tEncodeSize(tEncodeSVDropTbBatchReq, &reqNew, tlen, ret);
      void* buf = taosMemoryMalloc(tlen);
      if (NULL == buf) {
        taosArrayDestroy(reqNew.pArray);
        goto end;
      }
      SEncoder coderNew = {0};
      tEncoderInit(&coderNew, buf, tlen - sizeof(SMsgHead));
      tEncodeSVDropTbBatchReq(&coderNew, &reqNew);
      tEncoderClear(&coderNew);
      memcpy(pHead->body + sizeof(SMsgHead), buf, tlen);
      pHead->bodyLen = tlen + sizeof(SMsgHead);
      taosMemoryFree(buf);
      taosArrayDestroy(reqNew.pArray);
    }
  } else if (msgType == TDMT_VND_DELETE) {
    SDeleteRes req = {0};
    if (tDecodeDeleteRes(&coder, &req) < 0) {
      goto end;
    }
    realTbSuid = req.suid;
  }

L
Liu Jicong 已提交
181
end:
182 183 184 185
  tDecoderClear(&coder);
  return tbSuid == realTbSuid;
}

wmmhello's avatar
wmmhello 已提交
186
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
L
Liu Jicong 已提交
187
  int32_t code = 0;
wmmhello's avatar
wmmhello 已提交
188
  taosThreadMutexLock(&pHandle->pWalReader->mutex);
L
Liu Jicong 已提交
189 190 191
  int64_t offset = *fetchOffset;

  while (1) {
wmmhello's avatar
wmmhello 已提交
192 193 194
    if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
              pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
L
Liu Jicong 已提交
195 196 197 198 199
      *fetchOffset = offset - 1;
      code = -1;
      goto END;
    }

200 201 202
    tqDebug("vgId:%d, taosx get msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, offset,
            TMSG_INFO((*ppCkHead)->head.msgType));

L
Liu Jicong 已提交
203
    if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) {
wmmhello's avatar
wmmhello 已提交
204
      code = walFetchBody(pHandle->pWalReader, ppCkHead);
L
Liu Jicong 已提交
205 206 207 208 209 210 211 212 213 214

      if (code < 0) {
        *fetchOffset = offset;
        code = -1;
        goto END;
      }
      *fetchOffset = offset;
      code = 0;
      goto END;
    } else {
wmmhello's avatar
wmmhello 已提交
215
      if (pHandle->fetchMeta) {
L
Liu Jicong 已提交
216
        SWalCont* pHead = &((*ppCkHead)->head);
L
Liu Jicong 已提交
217
        if (IS_META_MSG(pHead->msgType)) {
wmmhello's avatar
wmmhello 已提交
218
          code = walFetchBody(pHandle->pWalReader, ppCkHead);
L
Liu Jicong 已提交
219 220 221 222 223
          if (code < 0) {
            *fetchOffset = offset;
            code = -1;
            goto END;
          }
L
Liu Jicong 已提交
224

L
Liu Jicong 已提交
225
          if (isValValidForTable(pHandle, pHead)) {
226 227 228
            *fetchOffset = offset;
            code = 0;
            goto END;
L
Liu Jicong 已提交
229 230 231
          } else {
            offset++;
            continue;
232
          }
L
Liu Jicong 已提交
233 234
        }
      }
wmmhello's avatar
wmmhello 已提交
235
      code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead);
L
Liu Jicong 已提交
236 237 238 239 240 241 242 243
      if (code < 0) {
        *fetchOffset = offset;
        code = -1;
        goto END;
      }
      offset++;
    }
  }
L
Liu Jicong 已提交
244
END:
wmmhello's avatar
wmmhello 已提交
245
  taosThreadMutexUnlock(&pHandle->pWalReader->mutex);
L
Liu Jicong 已提交
246 247 248
  return code;
}

L
Liu Jicong 已提交
249
STqReader* tqOpenReader(SVnode* pVnode) {
L
Liu Jicong 已提交
250
  STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader));
L
Liu Jicong 已提交
251
  if (pReader == NULL) {
L
Liu Jicong 已提交
252 253
    return NULL;
  }
L
Liu Jicong 已提交
254

L
Liu Jicong 已提交
255 256
  pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);
  if (pReader->pWalReader == NULL) {
L
Liu Jicong 已提交
257
    taosMemoryFree(pReader);
L
Liu Jicong 已提交
258 259
    return NULL;
  }
L
Liu Jicong 已提交
260 261

  pReader->pVnodeMeta = pVnode->pMeta;
L
Liu Jicong 已提交
262
  /*pReader->pMsg = NULL;*/
L
Liu Jicong 已提交
263 264 265 266 267 268 269 270 271 272 273 274
  pReader->ver = -1;
  pReader->pColIdList = NULL;
  pReader->cachedSchemaVer = 0;
  pReader->cachedSchemaSuid = 0;
  pReader->pSchema = NULL;
  pReader->pSchemaWrapper = NULL;
  pReader->tbIdHash = NULL;
  return pReader;
}

void tqCloseReader(STqReader* pReader) {
  // close wal reader
275 276 277
  if (pReader->pWalReader) {
    walCloseReader(pReader->pWalReader);
  }
L
Liu Jicong 已提交
278
  // free cached schema
279 280 281 282 283 284 285 286 287
  if (pReader->pSchema) {
    taosMemoryFree(pReader->pSchema);
  }
  if (pReader->pSchemaWrapper) {
    tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
  }
  if (pReader->pColIdList) {
    taosArrayDestroy(pReader->pColIdList);
  }
L
Liu Jicong 已提交
288
  // free hash
289
  taosHashCleanup(pReader->tbIdHash);
L
Liu Jicong 已提交
290 291 292
  taosMemoryFree(pReader);
}

H
Haojun Liao 已提交
293
int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
H
Haojun Liao 已提交
294
  // todo set the correct vgId
H
Haojun Liao 已提交
295
  tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
L
Liu Jicong 已提交
296
  if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
H
Haojun Liao 已提交
297
    tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
L
Liu Jicong 已提交
298
    return -1;
299
  } else {
300
    tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);
301
    return 0;
L
Liu Jicong 已提交
302
  }
L
Liu Jicong 已提交
303 304
}

L
Liu Jicong 已提交
305
int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
L
Liu Jicong 已提交
306
  bool fromProcessedMsg = pReader->msg2.msgStr != NULL;
L
Liu Jicong 已提交
307 308 309 310

  while (1) {
    if (!fromProcessedMsg) {
      if (walNextValidMsg(pReader->pWalReader) < 0) {
311 312 313 314 315 316
//        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
        if(pReader->pWalReader->curInvalid == 0){
          pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped;
        }else{
          pReader->ver = walGetLastVer(pReader->pWalReader->pWal);
        }
L
Liu Jicong 已提交
317
        ret->offset.type = TMQ_OFFSET__LOG;
318

L
Liu Jicong 已提交
319
        ret->offset.version = pReader->ver;
L
Liu Jicong 已提交
320
        ret->fetchType = FETCH_TYPE__NONE;
321
        tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version);
L
Liu Jicong 已提交
322 323
        return -1;
      }
wmmhello's avatar
wmmhello 已提交
324

325 326
      void*   body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
      int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
L
Liu Jicong 已提交
327
      int64_t ver = pReader->pWalReader->pHead->head.version;
wmmhello's avatar
wmmhello 已提交
328

L
Liu Jicong 已提交
329
      tqReaderSetSubmitReq2(pReader, body, bodyLen, ver);
L
Liu Jicong 已提交
330 331
    }

L
Liu Jicong 已提交
332
    while (tqNextDataBlock2(pReader)) {
L
Liu Jicong 已提交
333
      // TODO mem free
L
Liu Jicong 已提交
334
      memset(&ret->data, 0, sizeof(SSDataBlock));
335
      int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL);
L
Liu Jicong 已提交
336
      if (code != 0 || ret->data.info.rows == 0) {
L
Liu Jicong 已提交
337
        continue;
L
Liu Jicong 已提交
338 339
      }
      ret->fetchType = FETCH_TYPE__DATA;
L
Liu Jicong 已提交
340
      tqDebug("return data rows %d", ret->data.info.rows);
L
Liu Jicong 已提交
341 342 343 344
      return 0;
    }

    if (fromProcessedMsg) {
L
Liu Jicong 已提交
345 346
      ret->offset.type = TMQ_OFFSET__LOG;
      ret->offset.version = pReader->ver;
L
Liu Jicong 已提交
347
      ret->fetchType = FETCH_TYPE__SEP;
S
Shengliang Guan 已提交
348
      tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version);
L
Liu Jicong 已提交
349 350 351
      return 0;
    }
  }
L
Liu Jicong 已提交
352 353
}

L
Liu Jicong 已提交
354
#if 0
L
Liu Jicong 已提交
355
int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) {
L
Liu Jicong 已提交
356
  pReader->pMsg = pMsg;
L
Liu Jicong 已提交
357

wmmhello's avatar
wmmhello 已提交
358 359 360
//  if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
//  while (true) {
//    if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1;
361
//    tqDebug("submitnext vgId:%d, block:%p, dataLen:%d, len:%d, uid:%"PRId64, pWalReader->pWal->cfg.vgId, pReader->pBlock, pReader->msgIter.dataLen,
wmmhello's avatar
wmmhello 已提交
362 363 364
//            pReader->msgIter.len, pReader->msgIter.uid);
//    if (pReader->pBlock == NULL) break;
//  }
L
Liu Jicong 已提交
365

L
Liu Jicong 已提交
366 367 368
  if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
  pReader->ver = ver;
  memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter));
L
Liu Jicong 已提交
369
  return 0;
L
Liu Jicong 已提交
370
}
L
Liu Jicong 已提交
371 372 373
#endif

int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
374 375
  ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0));

L
Liu Jicong 已提交
376 377 378
  pReader->msg2.msgStr = msgStr;
  pReader->msg2.msgLen = msgLen;
  pReader->msg2.ver = ver;
L
Liu Jicong 已提交
379
  pReader->ver = ver;
L
Liu Jicong 已提交
380

L
Liu Jicong 已提交
381
  tqDebug("tq reader set msg %p %d", msgStr, msgLen);
L
Liu Jicong 已提交
382 383

  if (pReader->setMsg == 0) {
L
Liu Jicong 已提交
384 385
    SDecoder decoder;
    tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen);
L
Liu Jicong 已提交
386 387 388
    if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) {
      ASSERT(0);
    }
L
Liu Jicong 已提交
389
    tDecoderClear(&decoder);
L
Liu Jicong 已提交
390
    pReader->setMsg = 1;
L
Liu Jicong 已提交
391 392 393
  }
  return 0;
}
L
Liu Jicong 已提交
394

L
Liu Jicong 已提交
395
#if 0
L
Liu Jicong 已提交
396 397
bool tqNextDataBlock(STqReader* pReader) {
  if (pReader->pMsg == NULL) return false;
L
Liu Jicong 已提交
398
  while (1) {
L
Liu Jicong 已提交
399
    if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) {
L
Liu Jicong 已提交
400 401
      return false;
    }
L
Liu Jicong 已提交
402 403
    if (pReader->pBlock == NULL) {
      pReader->pMsg = NULL;
5
54liuyao 已提交
404 405
      return false;
    }
L
Liu Jicong 已提交
406

L
Liu Jicong 已提交
407
    if (pReader->tbIdHash == NULL) {
L
Liu Jicong 已提交
408 409
      return true;
    }
L
Liu Jicong 已提交
410
    void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
S
Shengliang Guan 已提交
411
    /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/
L
Liu Jicong 已提交
412
    if (ret != NULL) {
S
Shengliang Guan 已提交
413
      /*tqDebug("find   uid %" PRId64, pHandle->msgIter.uid);*/
L
Liu Jicong 已提交
414
      return true;
415 416 417 418
    }
  }
  return false;
}
L
Liu Jicong 已提交
419 420 421
#endif

bool tqNextDataBlock2(STqReader* pReader) {
422 423 424 425
  if (pReader->msg2.msgStr == NULL) {
    return false;
  }

L
Liu Jicong 已提交
426
  ASSERT(pReader->setMsg == 1);
L
Liu Jicong 已提交
427

L
Liu Jicong 已提交
428 429 430 431
  tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen,
          pReader->msg2.ver, pReader->nextBlk);

  int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
L
Liu Jicong 已提交
432
  while (pReader->nextBlk < blockSz) {
L
Liu Jicong 已提交
433
    SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
L
Liu Jicong 已提交
434 435
    ASSERT(pSubmitTbData->uid);

L
Liu Jicong 已提交
436 437 438 439 440 441
    if (pReader->tbIdHash == NULL) return true;

    void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
    if (ret != NULL) {
      return true;
    }
L
Liu Jicong 已提交
442
    pReader->nextBlk++;
L
Liu Jicong 已提交
443 444
  }

L
Liu Jicong 已提交
445 446
  tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE);
  pReader->setMsg = 0;
L
Liu Jicong 已提交
447 448 449 450 451 452 453 454
  pReader->nextBlk = 0;
  pReader->msg2.msgStr = NULL;

  return false;
}

bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) {
  if (pReader->msg2.msgStr == NULL) return false;
L
Liu Jicong 已提交
455
  ASSERT(pReader->setMsg == 1);
L
Liu Jicong 已提交
456

L
Liu Jicong 已提交
457
  int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
L
Liu Jicong 已提交
458
  while (pReader->nextBlk < blockSz) {
L
Liu Jicong 已提交
459
    SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
460
    if (filterOutUids == NULL) return true;
L
Liu Jicong 已提交
461

462
    void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t));
L
Liu Jicong 已提交
463 464 465
    if (ret == NULL) {
      return true;
    }
466
    pReader->nextBlk++;
L
Liu Jicong 已提交
467 468
  }

L
Liu Jicong 已提交
469 470
  tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE);
  pReader->setMsg = 0;
L
Liu Jicong 已提交
471 472 473 474 475
  pReader->nextBlk = 0;
  pReader->msg2.msgStr = NULL;

  return false;
}
476

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) {
  int32_t code;

  int32_t cnt = 0;
  for (int32_t i = 0; i < pSrc->nCols; i++) {
    cnt += mask[i];
  }

  pDst->nCols = cnt;
  pDst->pSchema = taosMemoryCalloc(cnt, sizeof(SSchema));
  if (pDst->pSchema == NULL) {
    return -1;
  }

  int32_t j = 0;
  for (int32_t i = 0; i < pSrc->nCols; i++) {
    if (mask[i]) {
      pDst->pSchema[j++] = pSrc->pSchema[i];
      SColumnInfoData colInfo =
          createColumnInfoData(pSrc->pSchema[i].type, pSrc->pSchema[i].bytes, pSrc->pSchema[i].colId);
      code = blockDataAppendColInfo(pBlock, &colInfo);
      if (code != 0) {
        return -1;
      }
    }
  }
  return 0;
}

L
Liu Jicong 已提交
506
#if 0
L
Liu Jicong 已提交
507
bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
508 509 510 511 512 513 514 515 516
  while (1) {
    if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
      return false;
    }
    if (pHandle->pBlock == NULL) return false;

    void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t));
    if (ret == NULL) {
      return true;
L
Liu Jicong 已提交
517 518 519 520 521
    }
  }
  return false;
}

L
Liu Jicong 已提交
522 523 524 525 526 527 528 529 530 531
int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) {
  //
  int32_t sversion = htonl(pReader->pBlock->sversion);
  if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
      pReader->cachedSchemaSuid != pReader->msgIter.suid) {
    taosMemoryFree(pReader->pSchema);
    pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
    if (pReader->pSchema == NULL) {
      tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
             "), version %d, possibly dropped table",
532
             pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion);
L
Liu Jicong 已提交
533 534 535 536 537 538 539 540 541
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }

    tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
    if (pReader->pSchemaWrapper == NULL) {
      tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
542
             pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer);
L
Liu Jicong 已提交
543 544 545 546 547 548 549 550 551 552 553 554 555 556
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }

    STSchema*       pTschema = pReader->pSchema;
    SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;

    int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
  }
  return 0;
}
#endif

557
int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) {
L
Liu Jicong 已提交
558
  int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
L
Liu Jicong 已提交
559 560
  ASSERT(pReader->nextBlk < blockSz);

L
Liu Jicong 已提交
561 562 563
  tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk);

  SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
L
Liu Jicong 已提交
564 565
  pReader->nextBlk++;

566
  if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData;
L
Liu Jicong 已提交
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
  int32_t sversion = pSubmitTbData->sver;
  int64_t suid = pSubmitTbData->suid;
  int64_t uid = pSubmitTbData->uid;
  pReader->lastBlkUid = uid;

  pBlock->info.id.uid = uid;
  pBlock->info.version = pReader->msg2.ver;

  if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) {
    taosMemoryFree(pReader->pSchema);
    pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1);
    if (pReader->pSchema == NULL) {
      tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
             "), version %d, possibly dropped table",
             pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion);
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }

    tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
    if (pReader->pSchemaWrapper == NULL) {
      tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
             pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }

    STSchema*       pTschema = pReader->pSchema;
    SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;

    int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);

    if (colNumNeed == 0) {
      int32_t colMeta = 0;
      while (colMeta < pSchemaWrapper->nCols) {
        SSchema*        pColSchema = &pSchemaWrapper->pSchema[colMeta];
        SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
        int32_t         code = blockDataAppendColInfo(pBlock, &colInfo);
        if (code != TSDB_CODE_SUCCESS) {
          goto FAIL;
        }
        colMeta++;
      }
    } else {
      if (colNumNeed > pSchemaWrapper->nCols) {
        colNumNeed = pSchemaWrapper->nCols;
      }

      int32_t colMeta = 0;
      int32_t colNeed = 0;
      while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
        SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
        col_id_t colIdSchema = pColSchema->colId;
        col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
        if (colIdSchema < colIdNeed) {
          colMeta++;
        } else if (colIdSchema > colIdNeed) {
          colNeed++;
        } else {
          SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
          int32_t         code = blockDataAppendColInfo(pBlock, &colInfo);
          if (code != TSDB_CODE_SUCCESS) {
            goto FAIL;
          }
          colMeta++;
          colNeed++;
        }
      }
    }

    int32_t numOfRows = 0;

    if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
      SArray*   pCols = pSubmitTbData->aCol;
      SColData* pCol = taosArrayGet(pCols, 0);
      numOfRows = pCol->nVal;
    } else {
      SArray* pRows = pSubmitTbData->aRowP;
      numOfRows = taosArrayGetSize(pRows);
    }

    if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      goto FAIL;
    }
    pBlock->info.rows = numOfRows;

    int32_t colActual = blockDataGetNumOfCols(pBlock);

    // convert and scan one block
    if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
      SArray* pCols = pSubmitTbData->aCol;
      int32_t numOfCols = taosArrayGetSize(pCols);
      int32_t targetIdx = 0;
      int32_t sourceIdx = 0;
      while (targetIdx < colActual) {
        ASSERT(sourceIdx < numOfCols);

        SColData*        pCol = taosArrayGet(pCols, sourceIdx);
        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
        SColVal          colVal;

        ASSERT(pCol->nVal == numOfRows);

        if (pCol->cid < pColData->info.colId) {
          sourceIdx++;
        } else if (pCol->cid == pColData->info.colId) {
          for (int32_t i = 0; i < pCol->nVal; i++) {
wmmhello's avatar
wmmhello 已提交
678
            tColDataGetValue(pCol, i, &colVal);
L
Liu Jicong 已提交
679
#if 0
L
Liu Jicong 已提交
680 681 682 683 684 685
            void* val = NULL;
            if (IS_STR_DATA_TYPE(colVal.type)) {
              val = colVal.value.pData;
            } else {
              val = &colVal.value.val;
            }
L
Liu Jicong 已提交
686
            if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
L
Liu Jicong 已提交
687 688
              goto FAIL;
            }
L
Liu Jicong 已提交
689 690
#endif
            if (IS_STR_DATA_TYPE(colVal.type)) {
L
Liu Jicong 已提交
691 692 693 694 695 696 697 698
              if (colVal.value.pData != NULL) {
                char val[65535 + 2];
                memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
                varDataSetLen(val, colVal.value.nData);
                if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                  goto FAIL;
                }
              } else {
X
Xiaoyu Wang 已提交
699
                colDataSetNULL(pColData, i);
L
Liu Jicong 已提交
700 701 702 703 704 705
              }
            } else {
              if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                goto FAIL;
              }
            }
L
Liu Jicong 已提交
706 707 708 709 710 711 712 713 714 715 716
          }
          sourceIdx++;
          targetIdx++;
        } else {
          ASSERT(0);
        }
      }
    } else {
      SArray* pRows = pSubmitTbData->aRowP;

      for (int32_t i = 0; i < numOfRows; i++) {
L
Liu Jicong 已提交
717
        SRow*   pRow = taosArrayGetP(pRows, i);
L
Liu Jicong 已提交
718 719 720 721 722
        int32_t sourceIdx = 0;

        for (int32_t j = 0; j < colActual; j++) {
          SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j);
          while (1) {
L
Liu Jicong 已提交
723
            ASSERT(sourceIdx < pTschema->numOfCols);
L
Liu Jicong 已提交
724 725 726 727 728 729 730 731

            SColVal colVal;
            tRowGet(pRow, pTschema, sourceIdx, &colVal);
            if (colVal.cid < pColData->info.colId) {
              sourceIdx++;
              continue;
            } else if (colVal.cid == pColData->info.colId) {
              if (IS_STR_DATA_TYPE(colVal.type)) {
L
Liu Jicong 已提交
732 733 734 735 736 737 738 739
                if (colVal.value.pData != NULL) {
                  char val[65535 + 2];
                  memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
                  varDataSetLen(val, colVal.value.nData);
                  if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                    goto FAIL;
                  }
                } else {
X
Xiaoyu Wang 已提交
740
                  colDataSetNULL(pColData, i);
L
Liu Jicong 已提交
741 742
                }
                /*val = colVal.value.pData;*/
L
Liu Jicong 已提交
743
              } else {
L
Liu Jicong 已提交
744 745 746
                if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                  goto FAIL;
                }
L
Liu Jicong 已提交
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
              }

              sourceIdx++;
              break;
            } else {
              ASSERT(0);
            }
          }
        }
      }
    }
  }

  return 0;

FAIL:
  blockDataFreeRes(pBlock);
  return -1;
}

#if 0
L
Liu Jicong 已提交
768
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
769
  // TODO: cache multiple schema
L
Liu Jicong 已提交
770 771 772 773
  int32_t sversion = htonl(pReader->pBlock->sversion);
  if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
      pReader->cachedSchemaSuid != pReader->msgIter.suid) {
    if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
774
    pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
L
Liu Jicong 已提交
775
    if (pReader->pSchema == NULL) {
S
Shengliang Guan 已提交
776
      tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
L
Liu Jicong 已提交
777
             pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
778
      pReader->cachedSchemaSuid = 0;
779
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
L
Liu Jicong 已提交
780 781
      return -1;
    }
L
Liu Jicong 已提交
782

L
Liu Jicong 已提交
783
    if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
784
    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
L
Liu Jicong 已提交
785
    if (pReader->pSchemaWrapper == NULL) {
S
Shengliang Guan 已提交
786
      tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
L
Liu Jicong 已提交
787
             pReader->msgIter.uid, pReader->cachedSchemaVer);
788
      pReader->cachedSchemaSuid = 0;
L
Liu Jicong 已提交
789 790 791
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }
L
Liu Jicong 已提交
792 793
    pReader->cachedSchemaVer = sversion;
    pReader->cachedSchemaSuid = pReader->msgIter.suid;
L
Liu Jicong 已提交
794 795
  }

L
Liu Jicong 已提交
796 797
  STSchema*       pTschema = pReader->pSchema;
  SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
L
Liu Jicong 已提交
798

L
Liu Jicong 已提交
799
  int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
L
Liu Jicong 已提交
800

L
Liu Jicong 已提交
801 802 803 804
  if (colNumNeed == 0) {
    int32_t colMeta = 0;
    while (colMeta < pSchemaWrapper->nCols) {
      SSchema*        pColSchema = &pSchemaWrapper->pSchema[colMeta];
805
      SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
L
Liu Jicong 已提交
806
      int32_t         code = blockDataAppendColInfo(pBlock, &colInfo);
807
      if (code != TSDB_CODE_SUCCESS) {
L
Liu Jicong 已提交
808
        goto FAIL;
L
Liu Jicong 已提交
809
      }
L
Liu Jicong 已提交
810
      colMeta++;
L
Liu Jicong 已提交
811 812 813 814 815 816 817 818 819 820 821
    }
  } else {
    if (colNumNeed > pSchemaWrapper->nCols) {
      colNumNeed = pSchemaWrapper->nCols;
    }

    int32_t colMeta = 0;
    int32_t colNeed = 0;
    while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
      SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
      col_id_t colIdSchema = pColSchema->colId;
L
Liu Jicong 已提交
822
      col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
L
Liu Jicong 已提交
823 824 825 826 827
      if (colIdSchema < colIdNeed) {
        colMeta++;
      } else if (colIdSchema > colIdNeed) {
        colNeed++;
      } else {
828
        SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
L
Liu Jicong 已提交
829
        int32_t         code = blockDataAppendColInfo(pBlock, &colInfo);
830
        if (code != TSDB_CODE_SUCCESS) {
L
Liu Jicong 已提交
831 832 833 834 835
          goto FAIL;
        }
        colMeta++;
        colNeed++;
      }
L
Liu Jicong 已提交
836 837
    }
  }
L
Liu Jicong 已提交
838

L
Liu Jicong 已提交
839
  if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) {
840
    terrno = TSDB_CODE_OUT_OF_MEMORY;
841 842 843 844
    goto FAIL;
  }

  int32_t colActual = blockDataGetNumOfCols(pBlock);
L
Liu Jicong 已提交
845

L
Liu Jicong 已提交
846 847 848 849
  STSRowIter iter = {0};
  tdSTSRowIterInit(&iter, pTschema);
  STSRow* row;
  int32_t curRow = 0;
850

L
Liu Jicong 已提交
851
  tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
852

H
Haojun Liao 已提交
853
  pBlock->info.id.uid = pReader->msgIter.uid;
L
Liu Jicong 已提交
854
  pBlock->info.rows = pReader->msgIter.numOfRows;
855
  pBlock->info.version = pReader->pMsg->version;
856
  pBlock->info.dataLoad = 1;
857

L
Liu Jicong 已提交
858
  while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
L
Liu Jicong 已提交
859 860
    tdSTSRowIterReset(&iter, row);
    // get all wanted col of that block
L
Liu Jicong 已提交
861
    for (int32_t i = 0; i < colActual; i++) {
862
      SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
L
Liu Jicong 已提交
863
      SCellVal         sVal = {0};
C
Cary Xu 已提交
864
      if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
L
Liu Jicong 已提交
865 866
        break;
      }
867
      if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) {
L
Liu Jicong 已提交
868
        goto FAIL;
L
Liu Jicong 已提交
869
      }
L
Liu Jicong 已提交
870 871 872
    }
    curRow++;
  }
L
Liu Jicong 已提交
873
  return 0;
874

L
Liu Jicong 已提交
875
FAIL:
876
  blockDataFreeRes(pBlock);
L
Liu Jicong 已提交
877
  return -1;
L
Liu Jicong 已提交
878
}
H
Hongze Cheng 已提交
879

880
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) {
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913
  int32_t sversion = htonl(pReader->pBlock->sversion);

  if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
      pReader->cachedSchemaSuid != pReader->msgIter.suid) {
    if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
    pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
    if (pReader->pSchema == NULL) {
      tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
             pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }

    if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
    if (pReader->pSchemaWrapper == NULL) {
      tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
             pReader->msgIter.uid, pReader->cachedSchemaVer);
      pReader->cachedSchemaSuid = 0;
      terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
      return -1;
    }
    pReader->cachedSchemaVer = sversion;
    pReader->cachedSchemaSuid = pReader->msgIter.suid;
  }

  STSchema*       pTschema = pReader->pSchema;
  SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;

  int32_t colAtMost = pSchemaWrapper->nCols;

  int32_t curRow = 0;
914
  int32_t lastRow = 0;
915 916

  char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
917
  if (assigned == NULL) return -1;
918 919 920 921 922 923 924 925 926 927

  tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
  STSRowIter iter = {0};
  tdSTSRowIterInit(&iter, pTschema);
  STSRow* row;

  while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
    bool buildNew = false;
    tdSTSRowIterReset(&iter, row);

928
    tqDebug("vgId:%d, row of block %d", pReader->pWalReader->pWal->cfg.vgId, curRow);
929 930 931 932 933
    for (int32_t i = 0; i < colAtMost; i++) {
      SCellVal sVal = {0};
      if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) {
        break;
      }
934
      tqDebug("vgId:%d, %d col, type %d", pReader->pWalReader->pWal->cfg.vgId, i, sVal.valType);
935 936 937 938 939 940 941 942 943 944 945 946 947
      if (curRow == 0) {
        assigned[i] = sVal.valType != TD_VTYPE_NONE;
        buildNew = true;
      } else {
        bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE;
        if (currentRowAssigned != assigned[i]) {
          assigned[i] = currentRowAssigned;
          buildNew = true;
        }
      }
    }

    if (buildNew) {
948 949 950 951 952 953 954 955 956
      if (taosArrayGetSize(blocks) > 0) {
        SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
        pLastBlock->info.rows = curRow - lastRow;
        lastRow = curRow;
      }
      SSDataBlock*    pBlock = createDataBlock();
      SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
      if (tqMaskBlock(pSW, pBlock, pSchemaWrapper, assigned) < 0) {
        blockDataDestroy(pBlock);
957 958
        goto FAIL;
      }
959 960 961 962 963 964
      SSDataBlock block = {0};
      assignOneDataBlock(&block, pBlock);
      blockDataDestroy(pBlock);

      tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
              (int32_t)taosArrayGetSize(block.pDataBlock));
965 966

      taosArrayPush(blocks, &block);
967
      taosArrayPush(schemas, &pSW);
968 969 970
    }

    SSDataBlock* pBlock = taosArrayGetLast(blocks);
H
Haojun Liao 已提交
971
    pBlock->info.id.uid = pReader->msgIter.uid;
972
    pBlock->info.rows = 0;
973 974
    pBlock->info.version = pReader->pMsg->version;

975 976 977
    tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId,
            (int32_t)taosArrayGetSize(blocks));

978 979 980 981 982
    if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) {
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      goto FAIL;
    }

983
    tdSTSRowIterReset(&iter, row);
984 985 986 987 988 989 990 991 992 993
    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
      SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
      SCellVal         sVal = {0};

      if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
        break;
      }

      ASSERT(sVal.valType != TD_VTYPE_NONE);

994
      if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
995 996
        goto FAIL;
      }
997 998
      tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i,
              sVal.valType == TD_VTYPE_NULL);
999 1000 1001
    }
    curRow++;
  }
1002 1003
  SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
  pLastBlock->info.rows = curRow - lastRow;
1004 1005 1006 1007 1008 1009 1010 1011

  taosMemoryFree(assigned);
  return 0;

FAIL:
  taosMemoryFree(assigned);
  return -1;
}
L
Liu Jicong 已提交
1012 1013
#endif

1014
int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
1015 1016 1017 1018 1019
  tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk);

  SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
  pReader->nextBlk++;

1020
  if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData;
1021 1022 1023 1024 1025 1026 1027 1028 1029
  int32_t sversion = pSubmitTbData->sver;
  int64_t suid = pSubmitTbData->suid;
  int64_t uid = pSubmitTbData->uid;
  pReader->lastBlkUid = uid;

  taosMemoryFree(pReader->pSchema);
  pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1);
  if (pReader->pSchema == NULL) {
    tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
1030
           "), version %d, possibly dropped table",
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
           pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion);
    pReader->cachedSchemaSuid = 0;
    terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
    return -1;
  }

  tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
  pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
  if (pReader->pSchemaWrapper == NULL) {
    tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
           pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);
    pReader->cachedSchemaSuid = 0;
    terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
    return -1;
  }

  STSchema*       pTschema = pReader->pSchema;
  SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
1049
  int32_t         numOfRows = 0;
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061

  if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
    SArray*   pCols = pSubmitTbData->aCol;
    SColData* pCol = taosArrayGet(pCols, 0);
    numOfRows = pCol->nVal;
  } else {
    SArray* pRows = pSubmitTbData->aRowP;
    numOfRows = taosArrayGetSize(pRows);
  }

  int32_t curRow = 0;
  int32_t lastRow = 0;
1062
  char*   assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
1063 1064 1065 1066 1067 1068 1069 1070 1071
  if (assigned == NULL) return -1;

  // convert and scan one block
  if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
    SArray* pCols = pSubmitTbData->aCol;
    int32_t numOfCols = taosArrayGetSize(pCols);
    for (int32_t i = 0; i < numOfRows; i++) {
      bool buildNew = false;

1072
      for (int32_t j = 0; j < numOfCols; j++) {
1073
        SColData* pCol = taosArrayGet(pCols, j);
1074
        SColVal   colVal;
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
        tColDataGetValue(pCol, i, &colVal);
        if (curRow == 0) {
          assigned[j] = !COL_VAL_IS_NONE(&colVal);
          buildNew = true;
        } else {
          bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);
          if (currentRowAssigned != assigned[j]) {
            assigned[j] = currentRowAssigned;
            buildNew = true;
          }
        }
      }

      if (buildNew) {
        if (taosArrayGetSize(blocks) > 0) {
          SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
          pLastBlock->info.rows = curRow - lastRow;
          lastRow = curRow;
        }

1095
        SSDataBlock     block = {0};
1096
        SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
1097
        if (pSW == NULL) {
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
          terrno = TSDB_CODE_OUT_OF_MEMORY;
          goto FAIL;
        }

        if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) {
          blockDataFreeRes(&block);
          tDeleteSSchemaWrapper(pSW);
          goto FAIL;
        }
        tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
                (int32_t)taosArrayGetSize(block.pDataBlock));

        block.info.id.uid = uid;
        block.info.version = pReader->msg2.ver;
        if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) {
          terrno = TSDB_CODE_OUT_OF_MEMORY;
          blockDataFreeRes(&block);
          tDeleteSSchemaWrapper(pSW);
          goto FAIL;
        }
        taosArrayPush(blocks, &block);
        taosArrayPush(schemas, &pSW);
      }

      SSDataBlock* pBlock = taosArrayGetLast(blocks);

      tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId,
              (int32_t)taosArrayGetSize(blocks));

      int32_t targetIdx = 0;
      int32_t sourceIdx = 0;
      int32_t colActual = blockDataGetNumOfCols(pBlock);
      while (targetIdx < colActual) {
        SColData*        pCol = taosArrayGet(pCols, sourceIdx);
        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
        SColVal          colVal;

        if (pCol->cid < pColData->info.colId) {
          sourceIdx++;
        } else if (pCol->cid == pColData->info.colId) {
          tColDataGetValue(pCol, i, &colVal);

          if (IS_STR_DATA_TYPE(colVal.type)) {
            if (colVal.value.pData != NULL) {
              char val[65535 + 2];
              memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
              varDataSetLen(val, colVal.value.nData);
              if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                goto FAIL;
              }
            } else {
X
Xiaoyu Wang 已提交
1149
              colDataSetNULL(pColData, curRow - lastRow);
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
            }
          } else {
            if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
              goto FAIL;
            }
          }
          sourceIdx++;
          targetIdx++;
        }
      }

      curRow++;
    }
  } else {
    SArray* pRows = pSubmitTbData->aRowP;
    for (int32_t i = 0; i < numOfRows; i++) {
1166 1167
      SRow* pRow = taosArrayGetP(pRows, i);
      bool  buildNew = false;
1168

1169
      for (int32_t j = 0; j < pTschema->numOfCols; j++) {
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
        SColVal colVal;
        tRowGet(pRow, pTschema, j, &colVal);
        if (curRow == 0) {
          assigned[j] = !COL_VAL_IS_NONE(&colVal);
          buildNew = true;
        } else {
          bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);
          if (currentRowAssigned != assigned[j]) {
            assigned[j] = currentRowAssigned;
            buildNew = true;
          }
        }
      }

      if (buildNew) {
        if (taosArrayGetSize(blocks) > 0) {
          SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
          pLastBlock->info.rows = curRow - lastRow;
          lastRow = curRow;
        }

1191
        SSDataBlock     block = {0};
1192
        SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
1193
        if (pSW == NULL) {
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
          terrno = TSDB_CODE_OUT_OF_MEMORY;
          goto FAIL;
        }

        if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) {
          blockDataFreeRes(&block);
          tDeleteSSchemaWrapper(pSW);
          goto FAIL;
        }
        tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
                (int32_t)taosArrayGetSize(block.pDataBlock));

        block.info.id.uid = uid;
        block.info.version = pReader->msg2.ver;
        if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) {
          terrno = TSDB_CODE_OUT_OF_MEMORY;
          blockDataFreeRes(&block);
          tDeleteSSchemaWrapper(pSW);
          goto FAIL;
        }
        taosArrayPush(blocks, &block);
        taosArrayPush(schemas, &pSW);
      }

      SSDataBlock* pBlock = taosArrayGetLast(blocks);

      tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId,
              (int32_t)taosArrayGetSize(blocks));

      int32_t targetIdx = 0;
      int32_t sourceIdx = 0;
      int32_t colActual = blockDataGetNumOfCols(pBlock);
      while (targetIdx < colActual) {
        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
1228
        SColVal          colVal;
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
        tRowGet(pRow, pTschema, sourceIdx, &colVal);

        if (colVal.cid < pColData->info.colId) {
          sourceIdx++;
        } else if (colVal.cid == pColData->info.colId) {
          if (IS_STR_DATA_TYPE(colVal.type)) {
            if (colVal.value.pData != NULL) {
              char val[65535 + 2];
              memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
              varDataSetLen(val, colVal.value.nData);
              if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
                goto FAIL;
              }
            } else {
X
Xiaoyu Wang 已提交
1243
              colDataSetNULL(pColData, curRow - lastRow);
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
            }
          } else {
            if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
              goto FAIL;
            }
          }
          sourceIdx++;
          targetIdx++;
        }
      }
      curRow++;
    }
L
Liu Jicong 已提交
1256
  }
1257 1258 1259 1260 1261 1262 1263

  SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
  pLastBlock->info.rows = curRow - lastRow;

  taosMemoryFree(assigned);
  return 0;

1264 1265 1266 1267 1268
FAIL:
  taosMemoryFree(assigned);
  return -1;
}

L
Liu Jicong 已提交
1269
void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; }
H
Hongze Cheng 已提交
1270

L
Liu Jicong 已提交
1271 1272 1273 1274
int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) {
  if (pReader->tbIdHash) {
    taosHashClear(pReader->tbIdHash);
  } else {
5
54liuyao 已提交
1275
    pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
H
Hongze Cheng 已提交
1276 1277
  }

L
Liu Jicong 已提交
1278
  if (pReader->tbIdHash == NULL) {
H
Hongze Cheng 已提交
1279 1280 1281 1282 1283 1284
    terrno = TSDB_CODE_OUT_OF_MEMORY;
    return -1;
  }

  for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
    int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
L
Liu Jicong 已提交
1285
    taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
H
Hongze Cheng 已提交
1286 1287 1288 1289 1290
  }

  return 0;
}

L
Liu Jicong 已提交
1291 1292
int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) {
  if (pReader->tbIdHash == NULL) {
5
54liuyao 已提交
1293
    pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
L
Liu Jicong 已提交
1294
    if (pReader->tbIdHash == NULL) {
H
Hongze Cheng 已提交
1295 1296 1297 1298 1299 1300 1301
      terrno = TSDB_CODE_OUT_OF_MEMORY;
      return -1;
    }
  }

  for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
    int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
L
Liu Jicong 已提交
1302
    taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
H
Hongze Cheng 已提交
1303 1304 1305 1306
  }

  return 0;
}
1307

L
Liu Jicong 已提交
1308
int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
L
Liu Jicong 已提交
1309 1310
  for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
    int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
L
Liu Jicong 已提交
1311
    taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t));
1312 1313 1314 1315
  }

  return 0;
}
L
Liu Jicong 已提交
1316 1317 1318 1319

int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
  void* pIter = NULL;
  while (1) {
1320
    pIter = taosHashIterate(pTq->pHandle, pIter);
H
Haojun Liao 已提交
1321 1322 1323 1324
    if (pIter == NULL) {
      break;
    }

L
Liu Jicong 已提交
1325 1326
    STqHandle* pExec = (STqHandle*)pIter;
    if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
1327
      int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd);
L
Liu Jicong 已提交
1328 1329 1330 1331
      if (code != 0) {
        tqError("update qualified table error for %s", pExec->subKey);
        continue;
      }
L
Liu Jicong 已提交
1332 1333 1334 1335 1336
    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
      if (!isAdd) {
        int32_t sz = taosArrayGetSize(tbUidList);
        for (int32_t i = 0; i < sz; i++) {
          int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
L
Liu Jicong 已提交
1337
          taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
L
Liu Jicong 已提交
1338 1339
        }
      }
1340 1341 1342 1343 1344 1345 1346 1347
    } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
      if (isAdd) {
        SArray*     qa = taosArrayInit(4, sizeof(tb_uid_t));
        SMetaReader mr = {0};
        metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
        for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) {
          uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i);

H
Haojun Liao 已提交
1348
          int32_t code = metaGetTableEntryByUidCache(&mr, *id);
1349
          if (code != TSDB_CODE_SUCCESS) {
L
Liu Jicong 已提交
1350
            tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
            continue;
          }

          tDecoderClear(&mr.coder);

          if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pExec->execHandle.execTb.suid) {
            tqDebug("table uid %" PRId64 " does not add to tq handle", *id);
            continue;
          }
          tqDebug("table uid %" PRId64 " add to tq handle", *id);
          taosArrayPush(qa, id);
        }
        metaReaderClear(&mr);
        if (taosArrayGetSize(qa) > 0) {
          tqReaderAddTbUidList(pExec->execHandle.pExecReader, qa);
        }
        taosArrayDestroy(qa);
      } else {
1369
        tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList);
1370
      }
L
Liu Jicong 已提交
1371 1372 1373
    }
  }
  while (1) {
L
Liu Jicong 已提交
1374
    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
L
Liu Jicong 已提交
1375
    if (pIter == NULL) break;
1376
    SStreamTask* pTask = *(SStreamTask**)pIter;
1377
    if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
L
Liu Jicong 已提交
1378
      int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
L
Liu Jicong 已提交
1379 1380 1381 1382
      if (code != 0) {
        tqError("update qualified table error for stream task %d", pTask->taskId);
        continue;
      }
L
Liu Jicong 已提交
1383 1384 1385 1386
    }
  }
  return 0;
}