[obkv] fix for batch get select empty columns return error

src/observer/table/ob_table_end_trans_cb.cpp

@@ -395,8 +395,8 @@ int ObTableLSExecuteEndTransCb::assign_ls_execute_result(const ObTableLSOpResult
     if (OB_FAIL(dst_tablet_result.prepare_allocate(single_res_cnt))) {
       LOG_WARN("fail to prepare allocatate single op result", K(ret), K(i), K(single_res_cnt));
     } else {
-      dst_tablet_result.assign_properties_names(&result_.get_rowkey_names());
-      dst_tablet_result.set_all_rowkey_names(&result_.get_properties_names());
+      dst_tablet_result.assign_properties_names(&result_.get_properties_names());
+      dst_tablet_result.set_all_rowkey_names(&result_.get_rowkey_names());
     }
     for (int64_t j = 0; OB_SUCC(ret) && j < single_res_cnt; j++) {
       const ObTableSingleOpResult &src_single_result = src_tablet_result.at(j);

src/observer/table/ob_table_ls_execute_processor.cpp

@@ -39,9 +39,10 @@ int ObTableLSExecuteP::before_process()
   int ret = OB_SUCCESS;
   const ObIArray<ObString>& all_rowkey_names = arg_.ls_op_.get_all_rowkey_names();
   const ObIArray<ObString>& all_properties_names = arg_.ls_op_.get_all_properties_names();
+  bool need_all_prop = arg_.ls_op_.need_all_prop_bitmap();
   if (OB_FAIL(result_.assign_rowkey_names(all_rowkey_names))) {
     LOG_WARN("fail to assign rowkey names", K(ret), K(all_rowkey_names));
-  } else if (OB_FAIL(result_.assign_properties_names(all_properties_names))) {
+  } else if (!need_all_prop && OB_FAIL(result_.assign_properties_names(all_properties_names))) {
     LOG_WARN("fail to assign properties names", K(ret), K(all_properties_names));
   } else {
     ret = ParentType::before_process();
@@ -171,6 +172,7 @@ int ObTableLSExecuteP::try_process()
   ObLSID ls_id = ls_op.get_ls_id();
   uint64_t table_id = ls_op.get_table_id();
   bool exist_global_index = false;
+  bool need_all_prop = arg_.ls_op_.need_all_prop_bitmap();
   table_id_ = table_id;  // init move response need
   if (OB_FAIL(init_schema_info(table_id))) {
     LOG_WARN("fail to init schema info", K(ret), K(table_id));
@@ -178,6 +180,17 @@ int ObTableLSExecuteP::try_process()
     LOG_WARN("fail to get ls id", K(ret));
   } else if (OB_FAIL(check_table_has_global_index(exist_global_index))) {
     LOG_WARN("fail to check global index", K(ret), K(table_id));
+  } else if (need_all_prop) {
+    ObSEArray<ObString, 8> all_prop_name;
+    const ObIArray<ObTableColumnInfo *>&column_info_array = schema_cache_guard_.get_column_info_array();
+    if (OB_FAIL(ObTableApiUtil::expand_all_columns(column_info_array, all_prop_name))) {
+      LOG_WARN("fail to expand all columns", K(ret));
+    } else if (OB_FAIL(result_.assign_properties_names(all_prop_name))) {
+      LOG_WARN("fail to assign property names to result", K(ret));
+    }
+  }
+
+  if (OB_FAIL(ret)) {
   } else if (OB_FAIL(start_trans(false, /* is_readonly */
                                  arg_.consistency_level_,
                                  ls_id,
@@ -501,8 +514,13 @@ int ObTableLSExecuteP::add_dict_and_bm_to_result_entity(const table::ObTableTabl
     const ObTableSingleOp &single_op = tablet_op.at(i);
     const ObTableSingleOpEntity &req_entity= single_op.get_entities().at(0);
     ObTableSingleOpEntity *result_entity = static_cast<ObTableSingleOpEntity *>(tablet_result.at(i).get_entity());
-    result_entity->set_dictionary(&ls_op.get_all_rowkey_names(), &ls_op.get_all_properties_names());
-    if (OB_FAIL(result_entity->construct_names_bitmap(req_entity))) {
+    bool need_rebuild_bitmap = arg_.ls_op_.need_all_prop_bitmap() && single_op.get_op_type() == ObTableOperationType::GET;
+    result_entity->set_dictionary(&result_.get_rowkey_names(), &result_.get_properties_names());
+    if (need_rebuild_bitmap) { // construct result entity bitmap based on all columns dict
+      if (OB_FAIL(result_entity->construct_names_bitmap_by_dict(req_entity))) {
+        LOG_WARN("fail to construct name bitmap by all columns", K(ret), K(i));
+      }
+    } else if (OB_FAIL(result_entity->construct_names_bitmap(req_entity))) { // directly use request bitmap as result bitmap
       LOG_WARN("fail to construct name bitmap", K(ret), K(i));
     }
   }

src/share/table/ob_table.cpp

@@ -2275,6 +2275,7 @@ int ObTableSingleOpEntity::deep_copy(common::ObIAllocator &allocator, const ObIT
 
     const ObTableBitMap *other_rowkey_bp = other.get_rowkey_names_bitmap();
     if (OB_ISNULL(other_rowkey_bp)) {
+      ret = OB_ERR_UNEXPECTED;
       LOG_WARN("failed to get_rowkey_names_bitmap", K(ret), K(other));
     } else if (OB_FAIL(rowkey_names_bp_.init_bitmap_size(other_rowkey_bp->get_valid_bits_num()))) {
       LOG_WARN("failed to init_bitmap_size", K(ret), KPC(other_rowkey_bp));
@@ -2291,6 +2292,7 @@ int ObTableSingleOpEntity::deep_copy(common::ObIAllocator &allocator, const ObIT
       if (OB_SUCC(ret)) {
         const ObTableBitMap *other_prop_name_bp = other.get_properties_names_bitmap();
         if (OB_ISNULL(other_prop_name_bp)) {
+          ret = OB_ERR_UNEXPECTED;
           LOG_WARN("failed to get_properties_names_bitmap", K(ret), K(other));
         } else if (OB_FAIL(properties_names_bp_.init_bitmap_size(other_prop_name_bp->get_valid_bits_num()))) {
           LOG_WARN("failed to init_bitmap_size", K(ret), KPC(other_prop_name_bp));
@@ -2317,6 +2319,7 @@ int ObTableSingleOpEntity::construct_names_bitmap(const ObITableEntity &req_enti
   if (this->get_rowkey_size() != 0) {
     const ObTableBitMap *other_rowkey_bp = req_entity.get_rowkey_names_bitmap();
     if (OB_ISNULL(other_rowkey_bp)) {
+      ret = OB_ERR_UNEXPECTED;
       LOG_WARN("failed to get_rowkey_names_bitmap", K(ret), K(req_entity));
     } else {
       rowkey_names_bp_ = *other_rowkey_bp;
@@ -2329,6 +2332,7 @@ int ObTableSingleOpEntity::construct_names_bitmap(const ObITableEntity &req_enti
     if (this->get_properties_count() != 0) {
       const ObTableBitMap *other_prop_name_bp = req_entity.get_properties_names_bitmap();
       if (OB_ISNULL(other_prop_name_bp)) {
+        ret = OB_ERR_UNEXPECTED;
         LOG_WARN("failed to get_properties_names_bitmap", K(ret), K(req_entity));
       } else {
         properties_names_bp_ = *other_prop_name_bp;
@@ -2340,6 +2344,66 @@ int ObTableSingleOpEntity::construct_names_bitmap(const ObITableEntity &req_enti
   return ret;
 }
 
+int ObTableSingleOpEntity::construct_names_bitmap_by_dict(const ObITableEntity &req_entity)
+{
+  int ret = OB_SUCCESS;
+  if (this->get_rowkey_size() != 0) {
+    const ObTableBitMap *other_rowkey_bp = req_entity.get_rowkey_names_bitmap();
+    if (OB_ISNULL(other_rowkey_bp)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("failed to get_rowkey_names_bitmap", K(ret), K(req_entity));
+    } else {
+      rowkey_names_bp_ = *other_rowkey_bp;
+    }
+  } else if (OB_FAIL(rowkey_names_bp_.init_bitmap_size(0))) {
+    LOG_WARN("failed to init bitmap size", K(ret));
+  }
+
+  if (OB_SUCC(ret)) {
+    if (this->get_properties_count() != 0) {
+      if (OB_FAIL(construct_properties_bitmap_by_dict(req_entity))) {
+        LOG_WARN("failed to construct properties bitmap by dict", K(ret));
+      }
+    } else if (OB_FAIL(properties_names_bp_.init_bitmap_size(0))) {
+      LOG_WARN("failed to init bitmap size", K(ret));
+    }
+  }
+  return ret;
+}
+
+int ObTableSingleOpEntity::construct_properties_bitmap_by_dict(const ObITableEntity &req_entity)
+{
+  int ret = OB_SUCCESS;
+  int64_t all_prop_count = OB_INVALID_SIZE; // all_prop_count = -1
+  if (OB_ISNULL(this->all_properties_names_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("all properties names is NULL", K(ret));
+  } else if (FALSE_IT(all_prop_count = this->all_properties_names_->count())) {
+  } else if (OB_FAIL(properties_names_bp_.init_bitmap(all_prop_count))) {
+    LOG_WARN("failed to init bitmap size", K(ret));
+  } else if (all_prop_count == this->get_properties_count()) {
+    // fastpath: propterties in entity has all columns
+    // and the result is from scan iterator, so we can ensure the property counts means all columns
+    if (OB_FAIL(properties_names_bp_.set_all_bits_true())) {
+      LOG_WARN("failed set all bits true", K(ret));
+    }
+  } else {
+    // slow path: find each property position in dict and set bitmap
+    const ObIArray<ObString> &prop_name = this->get_properties_names();
+    for (int64_t i = 0; i < prop_name.count() && OB_SUCC(ret); i++) {
+      int64_t idx = -1;
+      if (!has_exist_in_array(*all_properties_names_, prop_name.at(i), &idx)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("property name is not exist in properties name dict", K(ret),
+                  K(prop_name.at(i)), KPC(all_properties_names_), K(i));
+      } else if (OB_FAIL(properties_names_bp_.set(idx))) {
+        LOG_WARN("failed to set bitmap", K(ret), K(idx), K(prop_name.at(i)), K(properties_names_bp_));
+      }
+    }
+  }
+  return ret;
+}
+
 OB_DEF_SERIALIZE(ObTableSingleOpEntity)
 {
   int ret = OB_SUCCESS;
@@ -2593,6 +2657,31 @@ int ObTableBitMap::init_bitmap_size(int64_t valid_bits_num)
   return ret;
 }
 
+int ObTableBitMap::init_bitmap(int64_t valid_bits_num)
+{
+  int ret = OB_SUCCESS;
+  int64_t block_nums = get_need_blocks_num(valid_bits_num);
+  if (block_nums < 0) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("block_nums less than zero", K(ret), K(block_nums));
+  } else if (block_count_ >= 0) {
+    if (block_nums != block_count_) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_WARN("bitmap had init before with diffrent val", K(ret), K(block_count_), K(block_nums));
+    }
+  } else {
+    for (int64_t i = 0; i < block_nums && OB_SUCC(ret); i++) {
+      if (OB_FAIL(datas_.push_back(0))) {
+        LOG_WARN("failed to init block value", K(ret), K(i));
+      }
+    }
+    block_count_ = block_nums;
+    valid_bits_num_ = valid_bits_num;
+  }
+
+  return ret;
+}
+
 int ObTableBitMap::reset()
 {
   int ret = OB_SUCCESS;
@@ -2669,6 +2758,19 @@ int ObTableBitMap::set(int64_t bit_pos)
   return ret;
 }
 
+int ObTableBitMap::set_all_bits_true()
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(datas_.count() != block_count_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("datas_.count is not equal to block_count_", K(ret), K(block_count_), K(datas_.count()));
+  }
+  for (int64_t i = 0; i < block_count_ && OB_SUCC(ret); i++) {
+    datas_.at(i) = SIZE_TYPE_MAX;
+  }
+  return ret;
+}
+
 int64_t ObTableBitMap::get_serialize_size() const
 {
   int64_t len = 0;

src/share/table/ob_table.h

@@ -65,6 +65,7 @@ enum class ObTableEntityType
 class ObTableBitMap {
 public:
   typedef uint8_t size_type;
+  static const size_type SIZE_TYPE_MAX = UINT8_MAX;
   static const size_type BYTES_PER_BLOCK = sizeof(size_type);   // 1
   static const size_type BITS_PER_BLOCK = BYTES_PER_BLOCK * 8;  // 1 * 8 = 8
   static const size_type BLOCK_MOD_BITS = 3;                    // 2^3 = 8
@@ -79,8 +80,10 @@ public:
   int deserialize(const char *buf, const int64_t data_len, int64_t &pos);
   int serialize(char *buf, const int64_t buf_len, int64_t &pos) const;
   int64_t get_serialize_size() const;
-
+  // init bitmap wihtout allocating any blocks in datas_ (datas_.count() == 0)
   int init_bitmap_size(int64_t valid_bits_num);
+  // init bitmap with blocks in datas_
+  int init_bitmap(int64_t valid_bits_num);
   void clear()
   {
     datas_.reset();
@@ -100,6 +103,8 @@ public:
 
   int set(int64_t bit_pos);
 
+  int set_all_bits_true();
+
   OB_INLINE int64_t get_block_count() const
   {
     return block_count_;
@@ -1339,6 +1344,10 @@ public:
 
   virtual int deep_copy(common::ObIAllocator &allocator, const ObITableEntity &other) override;
 
+  int construct_names_bitmap_by_dict(const ObITableEntity& req_entity);
+
+  int construct_properties_bitmap_by_dict(const ObITableEntity& req_entity);
+
   static int construct_column_names(const ObTableBitMap &names_bit_map,
                                               const ObIArray<ObString> &all_column_names,
                                               ObIArray<ObString> &column_names);
@@ -1549,6 +1558,7 @@ public:
   OB_INLINE const ObIArray<ObString>& get_all_rowkey_names() {return rowkey_names_; }
   OB_INLINE const ObIArray<ObString>& get_all_properties_names() {return properties_names_; }
   OB_INLINE bool return_one_result() const { return return_one_result_; }
+  OB_INLINE bool need_all_prop_bitmap() const { return need_all_prop_bitmap_; }
 
   TO_STRING_KV(K_(ls_id),
                K_(table_name),
@@ -1560,7 +1570,8 @@ public:
                K_(rowkey_names),
                K_(properties_names),
                "tablet_ops_count_", tablet_ops_.count(),
-               K_(tablet_ops));
+               K_(tablet_ops),
+               K_(need_all_prop_bitmap));
 private:
   DISALLOW_COPY_AND_ASSIGN(ObTableLSOp);
   share::ObLSID ls_id_;
@@ -1576,7 +1587,8 @@ private:
       bool is_same_type_ : 1;
       bool is_same_properties_names_ : 1;
       bool return_one_result_ : 1;
-      uint64_t reserved : 61;
+      bool need_all_prop_bitmap_ : 1;
+      uint64_t reserved : 60;
     };
   };