Merge pull request #124 from Yanghello/embedding_ut

refactor embedding ut and fix ut testsuite

core/paddlefl_mpc/operators/mpc_lookup_table_v2_op.cc

@@ -41,7 +41,6 @@ public:
         auto table_dims = ctx->GetInputDim("W");
         auto ids_dims = ctx->GetInputDim("Ids");
         int ids_rank = ids_dims.size();
-        VLOG(5) << "ids rank is " << ids_rank << std::endl;
         PADDLE_ENFORCE_EQ(
             table_dims.size(), 3,
             "ShapeError: The dimensions of the 'mpc lookup table' must be 3. "

core/paddlefl_mpc/operators/mpc_mean_op.cc

@@ -85,9 +85,9 @@ protected:
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OPERATOR(mpc_mean, ops::MpcMeanOp, 
+REGISTER_OPERATOR(mpc_mean, ops::MpcMeanOp,
                  ops::MpcMeanOpMaker, 
-                 ops::MpcMeanOpInferVarType, 
+                 ops::MpcMeanOpInferVarType,
                  ops::MpcMeanOpGradMaker<paddle::framework::OpDesc>);
 
 REGISTER_OPERATOR(mpc_mean_grad, ops::MpcMeanGradOp);

core/paddlefl_mpc/operators/mpc_scale_op.cc

+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "core/paddlefl_mpc/operators/mpc_scale_op.h"
+
+#include <memory>
+#include <string>
+
+namespace paddle {
+namespace operators {
+
+class MpcScaleOp : public framework::OperatorWithKernel {
+ public:
+  MpcScaleOp(const std::string &type, const framework::VariableNameMap &inputs,
+          const framework::VariableNameMap &outputs,
+          const framework::AttributeMap &attrs)
+      : OperatorWithKernel(type, inputs, outputs, attrs) {}
+
+  void InferShape(framework::InferShapeContext *ctx) const override {
+    OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "scale");
+    OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "scale");
+
+    if (ctx->IsRuntime() && ctx->HasInput("ScaleTensor")) {
+      auto scale = ctx->Inputs("ScaleTensor");
+      PADDLE_ENFORCE_EQ(scale.size(), 1,
+                        platform::errors::InvalidArgument(
+                            "Input(ScaleTensor) size must be 1, "
+                            "but received size is %d.",
+                            scale.size()));
+    }
+
+    ctx->SetOutputDim("Out", ctx->GetInputDim("X"));
+    ctx->ShareLoD("X", /*->*/ "Out");
+  }
+
+  framework::OpKernelType GetExpectedKernelType(
+        const framework::ExecutionContext& ctx) const override {
+    auto input_data_type = OperatorWithKernel::IndicateVarDataType(
+                ctx, "X");
+    return framework::OpKernelType(input_data_type, ctx.GetPlace());
+  }
+};
+
+class MpcScaleOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+  void Make() override {
+    AddInput("X", "(Tensor) Input tensor of scale operator.");
+    AddInput("ScaleTensor",
+             "(Tensor) If provided, use this as "
+             "scale factor, this has a higher priority than "
+             "attr(scale), the shape of this tensor MUST BE 1.")
+        .AsDispensable();
+    AddOutput("Out", "(Tensor) Output tensor of scale operator.");
+    AddComment(R"DOC(
+**Scale operator**
+
+Apply scaling and bias addition to the input tensor.
+
+if bias_after_scale=True:
+
+$$Out = scale*X + bias$$
+
+else:
+
+$$Out = scale*(X + bias)$$
+)DOC");
+    AddAttr<float>("scale", "The scaling factor of the scale operator.")
+        .SetDefault(1.0);
+    AddAttr<float>("bias", "The bias of the scale operator.").SetDefault(0.0);
+    AddAttr<bool>(
+        "bias_after_scale",
+        "Apply bias addition after or before scaling. It is useful for "
+        "numeric stability in some circumstances.")
+        .SetDefault(true);
+  }
+};
+
+class MpcScaleOpVarTypeInference : public framework::VarTypeInference {
+ public:
+  void operator()(framework::InferVarTypeContext *ctx) const override {
+    ctx->SyncTypeAndDataType("X", "Out");
+  }
+
+};
+
+template <typename T>
+class MpcScaleGradMaker : public framework::SingleGradOpMaker<T> {
+ public:
+  using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
+
+  void Apply(GradOpPtr<T> grad_op) const override {
+    grad_op->SetType("mpc_scale");
+    grad_op->SetInput("X", this->OutputGrad("Out"));
+    if (this->HasInput("ScaleTensor") > 0) {
+      grad_op->SetInput("ScaleTensor", this->Input("ScaleTensor"));
+    }
+    grad_op->SetOutput("Out", this->InputGrad("X"));
+    grad_op->SetAttr("scale", this->GetAttr("scale"));
+    grad_op->SetAttr("bias", 0.0f);
+    grad_op->SetAttr("bias_after_scale", true);
+  }
+};
+
+//DECLARE_INPLACE_OP_INFERER(MpcScaleOpInplace, {"X", "Out"});
+}  // namespace operators
+}  // namespace paddle
+
+namespace ops = paddle::operators;
+
+REGISTER_OPERATOR(mpc_scale, ops::MpcScaleOp, ops::MpcScaleOpMaker,
+                  ops::MpcScaleGradMaker<paddle::framework::OpDesc>,
+                  ops::MpcScaleGradMaker<paddle::imperative::OpBase>,
+                  ops::MpcScaleOpVarTypeInference);
+REGISTER_OP_CPU_KERNEL(
+    mpc_scale,
+    ops::MpcScaleKernel<paddle::platform::CPUDeviceContext, int64_t>);

core/paddlefl_mpc/operators/mpc_scale_op.h

+/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "mpc_op.h"
+#include "paddle/fluid/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename T>
+static inline T GetAttrFromTensor(const framework::Tensor* tensor) {
+  const auto* tensor_data = tensor->data<T>();
+  framework::Tensor cpu_tensor;
+  if (platform::is_gpu_place(tensor->place())) {
+    TensorCopySync(*tensor, platform::CPUPlace(), &cpu_tensor);
+    tensor_data = cpu_tensor.data<T>();
+  }
+  return tensor_data[0];
+}
+
+template <typename DeviceContext, typename T>
+class MpcScaleKernel : public MpcOpKernel<T> {
+ public:
+  void ComputeImpl(const framework::ExecutionContext& ctx) const override {
+    auto* in_var = ctx.InputVar("X");
+    auto* in = framework::GetLoDTensorOrSelectedRowsValueFromVar(*in_var);
+
+    T bias = static_cast<T>(ctx.Attr<float>("bias") *
+                               std::pow(2, mpc::FIXED_POINTER_SCALING_FACTOR));
+    auto bias_after_scale = ctx.Attr<bool>("bias_after_scale");
+
+    auto scale = ctx.Attr<float>("scale");
+    if (ctx.HasInput("ScaleTensor")) {
+      auto* scale_tensor = ctx.Input<framework::Tensor>("ScaleTensor");
+      scale = GetAttrFromTensor<float>(scale_tensor);
+    }
+
+    auto* out_var = ctx.OutputVar("Out");
+    if (in_var->IsType<framework::SelectedRows>() && in_var != out_var) {
+      auto& in_slr = in_var->Get<framework::SelectedRows>();
+      auto* out_slr = out_var->GetMutable<framework::SelectedRows>();
+      out_slr->set_rows(in_slr.rows());
+      out_slr->set_height(in_slr.height());
+    }
+
+    auto* out =
+        framework::GetMutableLoDTensorOrSelectedRowsValueFromVar(out_var);
+    auto out_ptr = out->mutable_data<T>(in->place());
+
+    PADDLE_ENFORCE_EQ(in->dims(), out->dims(),
+                      "in and out should have the same dim");
+
+    PADDLE_ENFORCE_NOT_NULL(mpc::MpcInstance::mpc_protocol,
+              "Protocol %s is not yet created in MPC Protocol.");
+    auto mpc_operator = mpc::MpcInstance::mpc_instance()->mpc_protocol()->mpc_operators();
+    if (bias_after_scale) {
+      mpc_operator->scale(in, scale, out);
+      std::for_each(out_ptr, out_ptr + out->numel(), [&bias](T& i) { i += (bias / 3); });
+    } else {
+      const T* in_data = in->data<T>();
+      std::transform(in_data, in_data + in->numel(), out_ptr, [&bias](const T& in){ return in + bias / 3; });
+      mpc_operator->scale(in, scale, out);
+    }
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle

python/paddle_fl/mpc/input.py

@@ -117,13 +117,13 @@ def embedding(input,
 
     if is_sparse:
       warnings.warn("the process on sparse data is the same with dense data,"
-                   " this is, 'is_sparse' always set as 'False' in paddle_encrypted.")
+                   " that is, 'is_sparse' always set as 'False' in paddle_encrypted.")
     if is_distributed:
       warnings.warn("distributed deployment of paddle_encrypted has not been implemented."
-                   " this is, 'is_distributed' always set as 'False' in paddle_encrypted.")
+                   " that is, 'is_distributed' always set as 'False' in paddle_encrypted.")
     if padding_idx:
       warnings.warn("padding_idx is not supported in paddle_encrypted."
-                   " this is, 'padding_idx' always set as 'None' in paddle_encrypted.")
+                   " that is, 'padding_idx' always set as 'None' in paddle_encrypted.")
     helper = MpcLayerHelper('embedding', **locals())
     check_variable_and_dtype(input, 'input', ['int64'], 'paddle_encrypted.embedding')
     check_dtype(dtype, 'dtype', ['int64'],

python/paddle_fl/mpc/tests/unittests/run_test_example.sh

@@ -20,7 +20,7 @@ TEST_MODULES=("test_datautils_aby3"
 "test_op_fc"
 "test_op_relu"
 "test_op_compare"
-"test_input_embedding"
+"test_op_embedding"
 "test_op_softmax_with_cross_entropy"
 "test_op_batch_norm"
 "test_op_conv"

python/paddle_fl/mpc/tests/unittests/test_input_embedding.py

-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-This module test embedding op.
-
-"""
-import unittest
-from multiprocessing import Manager
-
-import numpy as np
-import paddle.fluid as fluid
-import paddle_fl.mpc as pfl_mpc
-import paddle_fl.mpc.data_utils.aby3 as aby3
-
-import test_op_base
-
-
-class TestInput(test_op_base.TestOpBase):
-
-    def gen_one_hot(self, input, depth):
-        """
-        example for generate mpc one hot tensor
-        """
-        data_var = fluid.data(name='input_data', shape=input.shape, dtype='int64')
-        ret1 = fluid.input.one_hot(input=data_var, depth=3)
-        exe =fluid.Executor(place=fluid.CPUPlace())
-        exe.run(fluid.default_startup_program())
-        data = exe.run(program=fluid.default_main_program(),feed={'input_data': input}, fetch_list=[ret1])
-        return data[0]
-
-    def embedding_op(self, **kwargs):
-        role = kwargs['role']
-        #data = kwargs['data']
-        data_normal = kwargs['data_normal']
-        data_share = kwargs['data_share'][role]
-
-        w_data = kwargs['w_data']
-        w_data_share = kwargs['w_data_share'][role]
-        return_results = kwargs['return_results']
-        expected_result = kwargs['expect_results']
-
-        pfl_mpc.init("aby3", role, "localhost", self.server, int(self.port))
-
-        w_param_attrs = fluid.ParamAttr(name='emb_weight',
-                                        learning_rate=0.5,
-                                        initializer=pfl_mpc.initializer.NumpyArrayInitializer(w_data_share),
-                                        trainable=True)
-        w_param_attrs1 = fluid.ParamAttr(name='emb_weight1',
-                                        learning_rate=0.5,
-                                        initializer=fluid.initializer.NumpyArrayInitializer(w_data),
-                                        trainable=True)
-        input_shape = np.delete(data_share.shape, 0, 0)
-        data1 = pfl_mpc.data(name='input', shape=input_shape, dtype='int64')
-        data2 = fluid.data(name='input1', shape=data_normal.shape, dtype='int64')
-
-        math_embedding = fluid.input.embedding(input=data2, size=w_data.shape, param_attr=w_param_attrs1, dtype='float32')
-
-        op_embedding = pfl_mpc.input.embedding(input=data1, size=(input_shape[1],input_shape[0]), param_attr=w_param_attrs, dtype='int64')
-
-        exe = fluid.Executor(place=fluid.CPUPlace())
-        exe.run(fluid.default_startup_program())
-
-        results = exe.run(feed={'input': data_share, 'input1': data_normal}, fetch_list=[op_embedding, math_embedding])
-
-        return_results.append(results[0])
-        expected_result.append(results[1])
-
-    def test_embedding_op(self):
-        data = np.array([[1, 0, 0], [0, 1, 0]])
-        data_normal = np.array([0, 1]).astype('int64')
-        w_data = np.array([[1, 2], [2, 3], [3, 4]])
-
-        # data = self.gen_one_hot(data_normal, w_data.shape[0]).astype('int64')
-
-        data_share = aby3.make_shares(np.array(data))
-        data_all3shares = np.array([aby3.get_aby3_shares(data_share, i) for i in range(3)])
-        w_data_share = aby3.make_shares(w_data)
-        w_data_all3shares = np.array([aby3.get_aby3_shares(w_data_share, i) for i in range(3)])
-
-        return_results = Manager().list()
-        expect_results = Manager().list()
-        ret = self.multi_party_run(target=self.embedding_op,
-                                   data=data,
-                                   data_normal=data_normal,
-                                   w_data=w_data,
-                                   data_share=data_all3shares,
-                                   w_data_share=w_data_all3shares,
-                                   return_results=return_results,
-                                   expect_results=expect_results)
-        self.assertEqual(ret[0], True)
-        revealed = aby3.reconstruct(np.array(return_results))
-        # print("reveal: ", revealed)
-        self.assertTrue(np.allclose(revealed, expect_results[0], atol=1e-4))
-
-    def test_mpc_one_hot(self):
-      data = np.array([0, 1]).astype('int64')
-      ret = self.gen_one_hot(data, 3)
-      mpc_one_hot = aby3.make_shares(ret)
-
-if __name__ == '__main__':
-    unittest.main()

python/paddle_fl/mpc/tests/unittests/test_op_embedding.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module test embedding op.
+
+"""
+import unittest
+from multiprocessing import Manager
+import numpy as np
+
+from op_test import OpTest
+import paddle_fl.mpc.data_utils.aby3 as aby3
+
+import paddle.fluid.core as core
+
+import paddle.fluid as fluid
+from paddle.fluid import Program, program_guard
+
+class TestLookupTableOp(OpTest):
+    def to_one_hot(self, x, depth):
+        out = np.zeros(shape=(np.product(x.shape), depth)).astype('float')
+        for i in range(np.product(x.shape)):
+            out[i, x[i]] = 1.0
+        return out
+
+    def setUp(self):
+        OpTest.setUp(self)
+        self.op_type = "mpc_lookup_table_v2"
+        self.dtype = "int64"
+        table = np.random.random((17, 31)).astype("float")
+        ids = np.random.randint(0, 17, 4).astype("int64")
+        share = lambda x: np.array([x * 65536/3] * 2).astype('int64')
+        ids_one_hot = self.to_one_hot(ids, table.shape[0])
+        mpc_table = share(table)
+        mpc_ids_one_hot = share(ids_one_hot)
+        self.inputs = {'W': mpc_table, 'Ids': mpc_ids_one_hot}
+        self.outputs = {'Out': table[ids]}
+
+    def test_check_output(self):
+        place = core.CPUPlace()
+        self.check_output_with_place(place, atol=1e-3)
+
+    def test_check_grad(self):
+        # set output type to 'int64'
+        # TODO: if not set outputs type to 'int64', exception will throw
+        self.outputs = {'Out': np.array([1]).astype('int64')}
+        place = core.CPUPlace()
+        self.check_grad_with_place(place, ['W'], 'Out', no_grad_set=set('Ids'), max_relative_error=0.01)
+
+
+
+if __name__ == "__main__":
+    unittest.main()
+

python/paddle_fl/mpc/tests/unittests/test_op_scale.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This module test scale op.
+
+"""
+import unittest
+from multiprocessing import Manager
+import numpy as np
+
+
+import test_op_base
+from op_test import OpTest
+import paddle_fl.mpc.data_utils.aby3 as aby3
+
+import paddle.fluid as fluid
+import paddle.fluid.core as core
+
+
+class TestScaleOp(OpTest):
+    def setUp(self):
+        self.op_type = "mpc_scale"
+        self.dtype = np.int64
+        self.init_dtype_type()
+        share = lambda x: np.array([x * 65536/3] * 2).astype('int64')
+        input_p = np.random.random((10, 10))
+        self.inputs = {'X': share(input_p).astype(self.dtype)}
+        self.attrs = {'scale': -2.3}
+        self.outputs = {
+            'Out': input_p * self.attrs['scale']
+        }
+
+    def init_dtype_type(self):
+        pass
+
+    def test_check_output(self):
+        place =  core.CPUPlace()
+        self.check_output_with_place(place, atol=1e-3,)
+
+    def test_check_grad(self):
+        place =  core.CPUPlace()
+        self.check_grad_with_place(place, ['X'], 'Out', max_relative_error=0.05)
+
+
+class TestScaleOpScaleVariable(OpTest):
+    def setUp(self):
+        self.op_type = "mpc_scale"
+        self.dtype = np.int64
+        self.init_dtype_type()
+        self.scale = -2.3
+        share = lambda x: np.array([x * 65536/3] * 2).astype('int64')
+        input_p = np.random.random((10, 10))
+        self.inputs = {
+            'X': share(input_p),
+            'ScaleTensor': np.array([self.scale]).astype('float')
+        }
+        self.attrs = {}
+        self.outputs = {'Out': input_p * self.scale}
+
+    def init_dtype_type(self):
+        pass
+
+    def test_check_output(self):
+        place =  core.CPUPlace()
+        self.check_output_with_place(place, atol=1e-3)
+
+    def test_check_grad(self):
+        place =  core.CPUPlace()
+        self.check_grad_with_place(place, ['X'], 'Out', max_relative_error=0.05)
+
+if __name__ == "__main__":
+    unittest.main()

python/paddle_fl/mpc/tests/unittests/testsuite.py

@@ -153,26 +153,26 @@ def append_loss_ops(block, output_names):
     else:
         avg_sum = []
         for cur_loss in mean_inputs:
-            cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[1])
+            cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[2, 1])
             op = block.append_op(
                 inputs={"X": [cur_loss]},
                 outputs={"Out": [cur_avg_loss]},
-                type="mean")
+                type="mpc_mean")
             op.desc.infer_var_type(block.desc)
             op.desc.infer_shape(block.desc)
             avg_sum.append(cur_avg_loss)
 
-        loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[1])
+        loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[2, 1])
         op_sum = block.append_op(
-            inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='sum')
+            inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='mpc_sum')
         op_sum.desc.infer_var_type(block.desc)
         op_sum.desc.infer_shape(block.desc)
 
-        loss = block.create_var(dtype=loss_sum.dtype, shape=[1])
+        loss = block.create_var(dtype=loss_sum.dtype, shape=[2, 1])
         op_loss = block.append_op(
             inputs={"X": loss_sum},
             outputs={"Out": loss},
-            type='scale',
+            type='mpc_scale',
             attrs={'scale': 1.0 / float(len(avg_sum))})
         op_loss.desc.infer_var_type(block.desc)
         op_loss.desc.infer_shape(block.desc)