diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index ef85ed69dbe87d4b5b1b1b5d6a04220a5266a635..5234b74daba88abab97d54d4b96d3364078b0df9 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -59,18 +59,6 @@ std::ostream &operator<<(std::ostream &os, const LoDTensor &t) { return os; } -LoD SliceLevels(const LoD &in, size_t level_begin, size_t level_end) { - LoD new_lod; - new_lod.reserve(level_end - level_begin); - for (size_t i = level_begin; i < level_end; i++) { - new_lod.emplace_back(in.at(i)); - } - // transform the lowest level to absolute offset. - LoD abs_offset_lod = ToAbsOffset(in); - new_lod.back() = abs_offset_lod[level_end - 1]; - return new_lod; -} - LoD SliceInLevel(const LoD &in, size_t level, size_t elem_begin, size_t elem_end) { PADDLE_ENFORCE_LT(level, in.size()); @@ -131,43 +119,6 @@ bool operator==(const LoD &a, const LoD &b) { return true; } -size_t LoDTensor::NumElements(size_t level, size_t idx) const { - PADDLE_ENFORCE_LT(level, NumLevels()); - PADDLE_ENFORCE_LT(idx, NumElements(level)); - return lod_[level][idx + 1] - lod_[level][idx]; -} - -size_t LoDTensor::NumInstancesInElement(size_t level, size_t idx) const { - PADDLE_ENFORCE_LT(level, NumLevels()); - PADDLE_ENFORCE_LT(idx, NumElements(level)); - auto abs_lod = ToAbsOffset(lod()); - size_t begin = abs_lod[level][idx]; - size_t end = abs_lod[level][idx + 1]; - return end - begin; -} - -void LoDTensor::ShrinkLevels(size_t level_begin, size_t level_end) { - auto new_lod = framework::SliceLevels(lod_, level_begin, level_end); - lod_ = new_lod; -} - -void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin, - size_t elem_end) { - PADDLE_ENFORCE_LT(level, NumLevels()); - PADDLE_ENFORCE_LT(elem_begin, NumElements(level)); - PADDLE_ENFORCE_LT(elem_end, NumElements(level) + 1); - - auto abs_lod = framework::ToAbsOffset(lod()); - auto new_lod = framework::SliceInLevel(lod_, level, elem_begin, elem_end); - lod_ = new_lod; - - // slice the underlying tensor - size_t begin = abs_lod[level][elem_begin]; - size_t end = abs_lod[level][elem_end]; - PADDLE_ENFORCE_LT(begin, end, "Cannot shrink, the result tensor is empty."); - ShareDataWith(Slice(begin, end)); -} - using LoDAndOffset = std::pair>; LoDAndOffset GetSubLoDAndAbsoluteOffset(const LoD &lod, size_t start_idx, size_t end_idx, size_t start_level) { diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index b27936c1986d13d33786035adeebe14da92b190e..4ec72428aa8b8b0c4a088002fcfa44c9ec892d36 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -60,14 +60,6 @@ using LoD = std::vector>; std::ostream& operator<<(std::ostream& os, const LoD& lod); std::ostream& operator<<(std::ostream& os, const LoDTensor& t); -/* - * Slice levels from a LoD. - * NOTE the lowest level should always be the absolute offsets of the underlying - * tensor instances. So if higher layers are sliced without the lowest level, - * the lower level of the sliced LoD will be transformed to the absolute offset. - */ -LoD SliceLevels(const LoD& in, size_t level_begin, size_t level_end); - LoD SliceInLevel(const LoD& in, size_t level, size_t elem_begin, size_t elem_end); /* @@ -116,35 +108,6 @@ class LoDTensor : public Tensor { return (lod_)[level].size() - 1; } - /* - * Number of lower-level elements. - * For example, a 2-level lod-tensor - * - * 0-th level | | - * 1-th level || ||| - * - * NumElements(0, 0) get 2 - * NumElements(0, 1) get 3 - */ - size_t NumElements(size_t level, size_t idx) const; - - /* - * Get the number of instances in the underlying tensor in the `idx`-th - * element. - */ - size_t NumInstancesInElement(size_t level, size_t idx) const; - - /* - * Shrink levels[level_begin:level_end] - */ - void ShrinkLevels(size_t level_begin, size_t level_end); - - /* - * Shrink elements of a level, [elem_begin: elem_end] - * @note: low performance in slice lod_. - */ - void ShrinkInLevel(size_t level, size_t elem_begin, size_t elem_end); - std::vector SplitLoDTensor( const std::vector places) const; diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 0868c1f6e695b8de6a755f167951f404de0942ca..52b87f48e5340ce4e265e2e77577f58daae039d2 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -54,92 +54,6 @@ class LoDTensorTester : public ::testing::Test { LoDTensor lod_tensor_; }; -TEST_F(LoDTensorTester, NumLevels) { ASSERT_EQ(lod_tensor_.NumLevels(), 3UL); } - -TEST_F(LoDTensorTester, NumElements) { - ASSERT_EQ(lod_tensor_.NumElements(0), 2UL); - ASSERT_EQ(lod_tensor_.NumElements(1), 3UL); - ASSERT_EQ(lod_tensor_.NumElements(2), 8UL); -} - -TEST_F(LoDTensorTester, NumElements2) { - ASSERT_EQ(lod_tensor_.NumElements(0, 0), 2UL); - ASSERT_EQ(lod_tensor_.NumElements(0, 1), 1UL); - ASSERT_EQ(lod_tensor_.NumElements(1, 1), 3UL); -} - -TEST_F(LoDTensorTester, ShrinkLevels) { - // slice 1 level - for (size_t level = 0; level < 3UL; ++level) { - LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.ShrinkLevels(level, level + 1); - ASSERT_EQ(new_lod_tensor.NumLevels(), 1UL); - ASSERT_EQ(new_lod_tensor.data(), lod_tensor_.data()); - } - // shrink 2 level - for (size_t level = 0; level < 2UL; ++level) { - LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.ShrinkLevels(level, level + 2); - // the lowest level's last element should be the tensor's batch_size. - ASSERT_EQ(new_lod_tensor.lod().back().back(), - lod_tensor_.lod().back().back()); - ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL); - ASSERT_EQ(new_lod_tensor.data(), lod_tensor_.data()); - } -} - -TEST_F(LoDTensorTester, ShrinkInLevel) { - size_t level = 0; - LoDTensor new_lod_tensor = lod_tensor_; - new_lod_tensor.ShrinkInLevel(level, 0, 1); - ASSERT_EQ(new_lod_tensor.NumLevels(), 3UL); - ASSERT_EQ(new_lod_tensor.NumElements(0), 1UL); - ASSERT_EQ(new_lod_tensor.NumElements(1), 2UL); - ASSERT_EQ(new_lod_tensor.NumElements(2), 5UL); - ASSERT_EQ(new_lod_tensor.dims()[0], 12); - for (int i = 0; i < 12 * 128; i++) { - ASSERT_EQ(new_lod_tensor.data()[i], i); - } - - level = 1; - new_lod_tensor = lod_tensor_; - new_lod_tensor.ShrinkInLevel(level, 1, 2); - ASSERT_EQ(new_lod_tensor.NumLevels(), 2UL); - ASSERT_EQ(new_lod_tensor.NumElements(0), 1UL); - ASSERT_EQ(new_lod_tensor.NumElements(1), 3UL); - ASSERT_EQ(new_lod_tensor.dims()[0], 7); - for (int i = 5 * 128; i < 12 * 128; i++) { - ASSERT_EQ(new_lod_tensor.data()[i - 5 * 128], i); - } - - LoDTensor t1; - t1.set_lod(lod_tensor_.lod()); - t1.ShareDataWith(lod_tensor_); - - LoDTensor t2; - t2.set_lod(lod_tensor_.lod()); - t2.ShareDataWith(lod_tensor_); - - t1.ShrinkInLevel(0, 1, 2); - t2.ShrinkInLevel(0, 0, 1); - EXPECT_NE(t1.data(), t2.data()); - EXPECT_NE(t1.data(), lod_tensor_.data()); -} - -TEST_F(LoDTensorTester, SerializeAndDeserialize) { - LoDTensor dst_tensor; - platform::CPUDeviceContext cpu_ctx((platform::CPUPlace())); - std::ostringstream oss; - SerializeToStream(oss, lod_tensor_, cpu_ctx); - std::istringstream iss(oss.str()); - DeserializeFromStream(iss, &dst_tensor, cpu_ctx); - float* dst_ptr = dst_tensor.mutable_data(platform::CPUPlace()); - for (int i = 0; i < kLodTensorSize; ++i) { - EXPECT_EQ(dst_ptr[i], i); - } - EXPECT_EQ(dst_tensor.lod(), lod_tensor_.lod()); -} - TEST(LodExpand, test) { LoD lod{{0, 2}}; LoDTensor tensor;