From e8d802159e7d8a9c4b89d0c4f7b9e8c4fa0d31d4 Mon Sep 17 00:00:00 2001 From: Yancey1989 Date: Tue, 24 Apr 2018 11:13:39 +0800 Subject: [PATCH] add lookup_sparse_table_op --- paddle/fluid/framework/lod_tensor_test.cc | 4 +- paddle/fluid/framework/selected_rows.cc | 10 +- paddle/fluid/framework/selected_rows.h | 8 +- paddle/fluid/framework/selected_rows_test.cc | 8 +- paddle/fluid/operators/detail/serde_test.cc | 2 +- .../fluid/operators/lookup_sparse_table_op.cc | 154 ++++++++++++++++++ .../unittests/test_lookup_sparse_table_op.py | 86 ++++++++++ 7 files changed, 257 insertions(+), 15 deletions(-) create mode 100644 paddle/fluid/operators/lookup_sparse_table_op.cc create mode 100644 python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py diff --git a/paddle/fluid/framework/lod_tensor_test.cc b/paddle/fluid/framework/lod_tensor_test.cc index 97ab98f09b1..77e5ec4c7dd 100644 --- a/paddle/fluid/framework/lod_tensor_test.cc +++ b/paddle/fluid/framework/lod_tensor_test.cc @@ -255,11 +255,11 @@ TEST(LoDTensor, RecordIO) { std::unique_ptr stream_ptr(stream); recordio::Scanner scanner(std::move(stream_ptr)); auto tensors = ReadFromRecordIO(&scanner, ctx); - ASSERT_EQ(tensors.size(), 2); + ASSERT_EQ(tensors.size(), static_cast(2)); assert_tensor_ok(tensors[0]); assert_tensor_ok(tensors[1]); tensors = ReadFromRecordIO(&scanner, ctx); - ASSERT_EQ(tensors.size(), 2); + ASSERT_EQ(tensors.size(), static_cast(2)); assert_tensor_ok(tensors[0]); assert_tensor_ok(tensors[1]); } diff --git a/paddle/fluid/framework/selected_rows.cc b/paddle/fluid/framework/selected_rows.cc index 794e7f74341..56cf6693caf 100644 --- a/paddle/fluid/framework/selected_rows.cc +++ b/paddle/fluid/framework/selected_rows.cc @@ -120,11 +120,11 @@ bool SelectedRows::HasKey(int64_t key) const { : true; } -std::vector SelectedRows::Get(std::vector keys, - framework::Tensor* value) const { +std::vector> SelectedRows::Get( + std::vector keys, framework::Tensor* value) const { PADDLE_ENFORCE(value->IsInitialized(), "The value tensor should be initialized."); - std::vector non_keys; + std::vector> non_keys_pair; int64_t value_width = value_->numel() / value_->dims()[0]; PADDLE_ENFORCE_EQ(value_width, value->numel() / value->dims()[0], "output tensor should have the same shape with table " @@ -133,7 +133,7 @@ std::vector SelectedRows::Get(std::vector keys, for (size_t i = 0; i < keys.size(); ++i) { int64_t index = Index(keys[i]); if (index == -1) { - non_keys.push_back(keys[i]); + non_keys_pair.push_back(std::make_pair(keys[i], static_cast(i))); } else { framework::VisitDataType( framework::ToDataType(value_->type()), @@ -141,7 +141,7 @@ std::vector SelectedRows::Get(std::vector keys, index * value_width, value_width)); } } - return non_keys; + return non_keys_pair; } bool SelectedRows::Set(int64_t key, const framework::Tensor& value) { diff --git a/paddle/fluid/framework/selected_rows.h b/paddle/fluid/framework/selected_rows.h index d6c9507b168..c27c927ee75 100644 --- a/paddle/fluid/framework/selected_rows.h +++ b/paddle/fluid/framework/selected_rows.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include #include #include "paddle/fluid/framework/lod_tensor.h" @@ -78,10 +79,11 @@ class SelectedRows { /* * @brief Get value by the key list, if the * - * @return a list of keys which does not exists in table + * @return a list of pair which contains the non-exists key and the index in + * the value */ - std::vector Get(std::vector keys, - framework::Tensor* tensor) const; + std::vector> Get(std::vector keys, + framework::Tensor* value) const; /* * @brief Set a key-value pair into the table. diff --git a/paddle/fluid/framework/selected_rows_test.cc b/paddle/fluid/framework/selected_rows_test.cc index 39fe6d92940..eefcaa5672c 100644 --- a/paddle/fluid/framework/selected_rows_test.cc +++ b/paddle/fluid/framework/selected_rows_test.cc @@ -59,7 +59,7 @@ TEST_F(SelectedRowsTester, SerializeAndDeseralize) { ASSERT_EQ(selected_rows_->GetCompleteDims(), dst_tensor.GetCompleteDims()); } -TEST_F(SelectedRowsTester, Table) { +TEST_F(SelectedRowsTester, SparseTable) { platform::CPUPlace cpu; SelectedRows table; // initialize a sparse table @@ -87,11 +87,11 @@ TEST_F(SelectedRowsTester, Table) { framework::Tensor get_value; get_value.mutable_data(framework::make_ddim({2, 100}), cpu); std::vector keys({non_key, key}); - auto non_keys = table.Get(keys, &get_value); + auto non_key_pairs = table.Get(keys, &get_value); ASSERT_EQ(get_value.data()[100], static_cast(10)); - ASSERT_EQ(non_keys.size(), static_cast(1)); - ASSERT_EQ(non_keys[0], non_key); + ASSERT_EQ(non_key_pairs.size(), static_cast(1)); + ASSERT_EQ(non_key_pairs[0].first, non_key); } } // namespace framework diff --git a/paddle/fluid/operators/detail/serde_test.cc b/paddle/fluid/operators/detail/serde_test.cc index 221d2f4c5b3..e9eaaf1cbcd 100644 --- a/paddle/fluid/operators/detail/serde_test.cc +++ b/paddle/fluid/operators/detail/serde_test.cc @@ -108,7 +108,7 @@ void RunSerdeTestSelectedRows(platform::Place place) { EXPECT_FLOAT_EQ(tensor_data2[i], 32.7); } for (size_t i = 0; i < rows2->size(); ++i) { - EXPECT_EQ(rows_data2[i], i); + EXPECT_EQ(rows_data2[i], static_cast(i)); } EXPECT_EQ(slr2->height(), 1000); } diff --git a/paddle/fluid/operators/lookup_sparse_table_op.cc b/paddle/fluid/operators/lookup_sparse_table_op.cc new file mode 100644 index 00000000000..249896993b5 --- /dev/null +++ b/paddle/fluid/operators/lookup_sparse_table_op.cc @@ -0,0 +1,154 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include + +#include "paddle/fluid/framework/data_type.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/platform/device_context.h" + +namespace paddle { +namespace operators { + +constexpr int64_t kNoPadding = -1; + +class LookupSparseTableInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of LookupSparseTableOp should not be null."); + auto shape_w = ctx->GetInputDim("W"); + auto shape_ids = ctx->GetInputDim("Ids"); + shape_w[0] = shape_ids.size(); + ctx->SetOutputDim("Out", shape_w); + } +}; + +class LookupSparseTableOp : public framework::OperatorBase { + public: + using framework::OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &dev_place) const override { + auto out_var = scope.FindVar(Output("Out")); + auto w_var = scope.FindVar(Input("W")); + auto ids_var = scope.FindVar(Input("Ids")); + unsigned int seed = static_cast(Attr("seed")); + float min = Attr("min"); + float max = Attr("max"); + + PADDLE_ENFORCE(out_var->IsType(), + "The type of Out var should be LodTensor."); + PADDLE_ENFORCE(w_var->IsType(), + "The type of W var should be SelectedRows."); + PADDLE_ENFORCE(ids_var->IsType(), + "The type of Ids var should be SelectedRows."); + auto &ids_t = ids_var->Get(); + auto out_t = out_var->GetMutable(); + auto w_t = w_var->GetMutable(); + auto keys = ids_t.rows(); + + // TODO(Yancey1989): support CUDA Place for the sparse table + platform::CPUPlace cpu; + auto out_shape = w_t->value().dims(); + out_shape[0] = keys.size(); + out_t->Resize(out_shape); + out_t->mutable_data(cpu, w_t->value().type()); + + PADDLE_ENFORCE_EQ(framework::ToDataType(w_t->value().type()), + framework::proto::VarType::FP32, + "The sparse table only support FP32"); + + auto non_keys_pair = w_t->Get(keys, out_t); + auto value_shape = w_t->value().dims(); + value_shape[0] = 1; + for (const auto &it : non_keys_pair) { + const auto key = it.first; + const auto index = it.second; + framework::Tensor value; + value.Resize(value_shape); + auto data = value.mutable_data(cpu); + + std::minstd_rand engine; + engine.seed(seed); + std::uniform_real_distribution dist(min, max); + int64_t size = value.numel(); + for (int64_t i = 0; i < size; ++i) { + data[i] = dist(engine); + } + w_t->Set(key, value); + memory::Copy(cpu, out_t->mutable_data(cpu) + index * value.numel(), + cpu, value.data(), value.numel() * sizeof(float)); + } + } +}; + +class LookupSparseTableOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LookupSparseTableOpMaker(OpProto *proto, OpAttrChecker *op_checker) + : framework::OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("W", + "(SelectedRows) The input represents embedding table, " + "which is a learnable parameter."); + AddInput("Ids", + "(SelectedRows) Ids's type should be SelectedRows " + "the rows of Ids contains the Ids to be looked up in W."); + AddOutput("Out", + "(SelectedRows) The lookup results, which have the " + "same type as W."); + AddAttr("padding_idx", + "(int64, default -1) " + "If the value is -1, it makes no effect to lookup. " + "Otherwise the given value indicates padding the output " + "with zeros whenever lookup encounters it in Ids.") + .SetDefault(kNoPadding); + AddAttr("min", + "(float, default -1.0) " + "Minimum value of uniform random") + .SetDefault(-1.0f); + AddAttr("max", + "(float, default 1.0) " + "Maximun value of uniform random") + .SetDefault(1.0f); + AddAttr("seed", + "(int, default 0) " + "Random seed used for generating samples. " + "0 means use a seed generated by the system." + "Note that if seed is not 0, this operator will always " + "generate the same random numbers every time.") + .SetDefault(0); + AddComment(R"DOC( +Lookup Sprase Tablel Operator. + +This operator is used to perform lookup on parameter W, +then concatenated into a sparse tensor. + +The type of Ids(Input) is SelectedRows, the rows of Ids contains +the ids to be looked up in W; +if the Id is not in the sparse table, this operator will return a +random value and set the value into the table for the next looking up. + +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(lookup_sparse_table, ops::LookupSparseTableOp, + ops::LookupSparseTableInferShape, + ops::LookupSparseTableOpMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py new file mode 100644 index 00000000000..6c339cba83c --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py @@ -0,0 +1,86 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest +import paddle.fluid.core as core +from paddle.fluid.op import Operator + + +def output_hist(out): + hist, _ = np.histogram(out, range=(-5, 10)) + hist = hist.astype("float32") + hist /= float(out.size) + prob = 0.1 * np.ones((10)) + return hist, prob + + +class TestLookupSpraseTable(OpTest): + def check_with_place(self, place): + scope = core.Scope() + + # create and initialize Id Variable + ids = scope.var("Ids").get_selected_rows() + ids_array = [0, 2, 3, 5, 100] + ids.set_rows(ids_array) + + # create and initialize W Variable + rows = [0, 1, 2, 3, 4, 5, 6] + row_numel = 10000 + + w_selected_rows = scope.var('W').get_selected_rows() + w_selected_rows.set_height(len(rows)) + w_selected_rows.set_rows(rows) + w_array = np.ones((len(rows), row_numel)).astype("float32") + for i in range(len(rows)): + w_array[i] *= i + w_tensor = w_selected_rows.get_tensor() + w_tensor.set(w_array, place) + + # create Out Variable + out_tensor = scope.var('Out').get_tensor() + + # create and run lookup_table operator + lookup_table = Operator( + "lookup_sparse_table", + W='W', + Ids='Ids', + Out='Out', + min=-5.0, + max=10.0, + seed=10) + lookup_table.run(scope, place) + + # get result from Out + result_array = np.array(out_tensor) + # all(): return True if all elements of the iterable are true (or if the iterable is empty) + for idx, row in enumerate(ids_array[:-2]): + assert (row == result_array[idx]).all() + + # check the random value + hist, prob = output_hist(result_array[-1]) + self.assertTrue( + np.allclose( + hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + + def test_w_is_selected_rows(self): + places = [core.CPUPlace()] + # currently only support CPU + for place in places: + self.check_with_place(place) + + +if __name__ == "__main__": + unittest.main() -- GitLab