diff --git a/core/paddlefl_mpc/operators/mpc_lookup_table_v2_op.cc b/core/paddlefl_mpc/operators/mpc_lookup_table_v2_op.cc index 2bada784621b94da335aa41facafacd7edb20e98..b57349427111af3d9669c4d0d1e24ad583174fd3 100644 --- a/core/paddlefl_mpc/operators/mpc_lookup_table_v2_op.cc +++ b/core/paddlefl_mpc/operators/mpc_lookup_table_v2_op.cc @@ -41,7 +41,6 @@ public: auto table_dims = ctx->GetInputDim("W"); auto ids_dims = ctx->GetInputDim("Ids"); int ids_rank = ids_dims.size(); - VLOG(5) << "ids rank is " << ids_rank << std::endl; PADDLE_ENFORCE_EQ( table_dims.size(), 3, "ShapeError: The dimensions of the 'mpc lookup table' must be 3. " diff --git a/core/paddlefl_mpc/operators/mpc_mean_op.cc b/core/paddlefl_mpc/operators/mpc_mean_op.cc index 626fb204dd49a54d805fd8428ab52cb2fb4fa407..5b267519342c0c2bd36dc585fb5bf2f8c9e5fb99 100644 --- a/core/paddlefl_mpc/operators/mpc_mean_op.cc +++ b/core/paddlefl_mpc/operators/mpc_mean_op.cc @@ -85,9 +85,9 @@ protected: } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(mpc_mean, ops::MpcMeanOp, +REGISTER_OPERATOR(mpc_mean, ops::MpcMeanOp, ops::MpcMeanOpMaker, - ops::MpcMeanOpInferVarType, + ops::MpcMeanOpInferVarType, ops::MpcMeanOpGradMaker); REGISTER_OPERATOR(mpc_mean_grad, ops::MpcMeanGradOp); diff --git a/core/paddlefl_mpc/operators/mpc_scale_op.cc b/core/paddlefl_mpc/operators/mpc_scale_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..4d421044a17a3b96c60de2c5136049781c81bcd4 --- /dev/null +++ b/core/paddlefl_mpc/operators/mpc_scale_op.cc @@ -0,0 +1,127 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "core/paddlefl_mpc/operators/mpc_scale_op.h" + +#include +#include + +namespace paddle { +namespace operators { + +class MpcScaleOp : public framework::OperatorWithKernel { + public: + MpcScaleOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + void InferShape(framework::InferShapeContext *ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "scale"); + OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "scale"); + + if (ctx->IsRuntime() && ctx->HasInput("ScaleTensor")) { + auto scale = ctx->Inputs("ScaleTensor"); + PADDLE_ENFORCE_EQ(scale.size(), 1, + platform::errors::InvalidArgument( + "Input(ScaleTensor) size must be 1, " + "but received size is %d.", + scale.size())); + } + + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Out"); + } + + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + auto input_data_type = OperatorWithKernel::IndicateVarDataType( + ctx, "X"); + return framework::OpKernelType(input_data_type, ctx.GetPlace()); + } +}; + +class MpcScaleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", "(Tensor) Input tensor of scale operator."); + AddInput("ScaleTensor", + "(Tensor) If provided, use this as " + "scale factor, this has a higher priority than " + "attr(scale), the shape of this tensor MUST BE 1.") + .AsDispensable(); + AddOutput("Out", "(Tensor) Output tensor of scale operator."); + AddComment(R"DOC( +**Scale operator** + +Apply scaling and bias addition to the input tensor. + +if bias_after_scale=True: + +$$Out = scale*X + bias$$ + +else: + +$$Out = scale*(X + bias)$$ +)DOC"); + AddAttr("scale", "The scaling factor of the scale operator.") + .SetDefault(1.0); + AddAttr("bias", "The bias of the scale operator.").SetDefault(0.0); + AddAttr( + "bias_after_scale", + "Apply bias addition after or before scaling. It is useful for " + "numeric stability in some circumstances.") + .SetDefault(true); + } +}; + +class MpcScaleOpVarTypeInference : public framework::VarTypeInference { + public: + void operator()(framework::InferVarTypeContext *ctx) const override { + ctx->SyncTypeAndDataType("X", "Out"); + } + +}; + +template +class MpcScaleGradMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + void Apply(GradOpPtr grad_op) const override { + grad_op->SetType("mpc_scale"); + grad_op->SetInput("X", this->OutputGrad("Out")); + if (this->HasInput("ScaleTensor") > 0) { + grad_op->SetInput("ScaleTensor", this->Input("ScaleTensor")); + } + grad_op->SetOutput("Out", this->InputGrad("X")); + grad_op->SetAttr("scale", this->GetAttr("scale")); + grad_op->SetAttr("bias", 0.0f); + grad_op->SetAttr("bias_after_scale", true); + } +}; + +//DECLARE_INPLACE_OP_INFERER(MpcScaleOpInplace, {"X", "Out"}); +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR(mpc_scale, ops::MpcScaleOp, ops::MpcScaleOpMaker, + ops::MpcScaleGradMaker, + ops::MpcScaleGradMaker, + ops::MpcScaleOpVarTypeInference); +REGISTER_OP_CPU_KERNEL( + mpc_scale, + ops::MpcScaleKernel); diff --git a/core/paddlefl_mpc/operators/mpc_scale_op.h b/core/paddlefl_mpc/operators/mpc_scale_op.h new file mode 100644 index 0000000000000000000000000000000000000000..088610a015d4be6bdb06da6834048b16585f7351 --- /dev/null +++ b/core/paddlefl_mpc/operators/mpc_scale_op.h @@ -0,0 +1,81 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "mpc_op.h" +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +static inline T GetAttrFromTensor(const framework::Tensor* tensor) { + const auto* tensor_data = tensor->data(); + framework::Tensor cpu_tensor; + if (platform::is_gpu_place(tensor->place())) { + TensorCopySync(*tensor, platform::CPUPlace(), &cpu_tensor); + tensor_data = cpu_tensor.data(); + } + return tensor_data[0]; +} + +template +class MpcScaleKernel : public MpcOpKernel { + public: + void ComputeImpl(const framework::ExecutionContext& ctx) const override { + auto* in_var = ctx.InputVar("X"); + auto* in = framework::GetLoDTensorOrSelectedRowsValueFromVar(*in_var); + + T bias = static_cast(ctx.Attr("bias") * + std::pow(2, mpc::FIXED_POINTER_SCALING_FACTOR)); + auto bias_after_scale = ctx.Attr("bias_after_scale"); + + auto scale = ctx.Attr("scale"); + if (ctx.HasInput("ScaleTensor")) { + auto* scale_tensor = ctx.Input("ScaleTensor"); + scale = GetAttrFromTensor(scale_tensor); + } + + auto* out_var = ctx.OutputVar("Out"); + if (in_var->IsType() && in_var != out_var) { + auto& in_slr = in_var->Get(); + auto* out_slr = out_var->GetMutable(); + out_slr->set_rows(in_slr.rows()); + out_slr->set_height(in_slr.height()); + } + + auto* out = + framework::GetMutableLoDTensorOrSelectedRowsValueFromVar(out_var); + auto out_ptr = out->mutable_data(in->place()); + + PADDLE_ENFORCE_EQ(in->dims(), out->dims(), + "in and out should have the same dim"); + + PADDLE_ENFORCE_NOT_NULL(mpc::MpcInstance::mpc_protocol, + "Protocol %s is not yet created in MPC Protocol."); + auto mpc_operator = mpc::MpcInstance::mpc_instance()->mpc_protocol()->mpc_operators(); + if (bias_after_scale) { + mpc_operator->scale(in, scale, out); + std::for_each(out_ptr, out_ptr + out->numel(), [&bias](T& i) { i += (bias / 3); }); + } else { + const T* in_data = in->data(); + std::transform(in_data, in_data + in->numel(), out_ptr, [&bias](const T& in){ return in + bias / 3; }); + mpc_operator->scale(in, scale, out); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle_fl/mpc/input.py b/python/paddle_fl/mpc/input.py index 048247c7ae3a5c0588066a525cd01e88bd4cd3e2..7a8d74e07a5f5b9516d7267cf4250a17cdea82a2 100644 --- a/python/paddle_fl/mpc/input.py +++ b/python/paddle_fl/mpc/input.py @@ -117,13 +117,13 @@ def embedding(input, if is_sparse: warnings.warn("the process on sparse data is the same with dense data," - " this is, 'is_sparse' always set as 'False' in paddle_encrypted.") + " that is, 'is_sparse' always set as 'False' in paddle_encrypted.") if is_distributed: warnings.warn("distributed deployment of paddle_encrypted has not been implemented." - " this is, 'is_distributed' always set as 'False' in paddle_encrypted.") + " that is, 'is_distributed' always set as 'False' in paddle_encrypted.") if padding_idx: warnings.warn("padding_idx is not supported in paddle_encrypted." - " this is, 'padding_idx' always set as 'None' in paddle_encrypted.") + " that is, 'padding_idx' always set as 'None' in paddle_encrypted.") helper = MpcLayerHelper('embedding', **locals()) check_variable_and_dtype(input, 'input', ['int64'], 'paddle_encrypted.embedding') check_dtype(dtype, 'dtype', ['int64'], diff --git a/python/paddle_fl/mpc/tests/unittests/run_test_example.sh b/python/paddle_fl/mpc/tests/unittests/run_test_example.sh index 3271ea21d24dbb9f1edd28a9fab1c32c4ab01c4e..a8c8db06756b777e92a9a902b734887125ce3cbf 100644 --- a/python/paddle_fl/mpc/tests/unittests/run_test_example.sh +++ b/python/paddle_fl/mpc/tests/unittests/run_test_example.sh @@ -20,7 +20,7 @@ TEST_MODULES=("test_datautils_aby3" "test_op_fc" "test_op_relu" "test_op_compare" -"test_input_embedding" +"test_op_embedding" "test_op_softmax_with_cross_entropy" "test_op_batch_norm" "test_op_conv" diff --git a/python/paddle_fl/mpc/tests/unittests/test_input_embedding.py b/python/paddle_fl/mpc/tests/unittests/test_input_embedding.py deleted file mode 100644 index db417b1d1f7fabb875304b95daa5908df88201c7..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/tests/unittests/test_input_embedding.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This module test embedding op. - -""" -import unittest -from multiprocessing import Manager - -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc -import paddle_fl.mpc.data_utils.aby3 as aby3 - -import test_op_base - - -class TestInput(test_op_base.TestOpBase): - - def gen_one_hot(self, input, depth): - """ - example for generate mpc one hot tensor - """ - data_var = fluid.data(name='input_data', shape=input.shape, dtype='int64') - ret1 = fluid.input.one_hot(input=data_var, depth=3) - exe =fluid.Executor(place=fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - data = exe.run(program=fluid.default_main_program(),feed={'input_data': input}, fetch_list=[ret1]) - return data[0] - - def embedding_op(self, **kwargs): - role = kwargs['role'] - #data = kwargs['data'] - data_normal = kwargs['data_normal'] - data_share = kwargs['data_share'][role] - - w_data = kwargs['w_data'] - w_data_share = kwargs['w_data_share'][role] - return_results = kwargs['return_results'] - expected_result = kwargs['expect_results'] - - pfl_mpc.init("aby3", role, "localhost", self.server, int(self.port)) - - w_param_attrs = fluid.ParamAttr(name='emb_weight', - learning_rate=0.5, - initializer=pfl_mpc.initializer.NumpyArrayInitializer(w_data_share), - trainable=True) - w_param_attrs1 = fluid.ParamAttr(name='emb_weight1', - learning_rate=0.5, - initializer=fluid.initializer.NumpyArrayInitializer(w_data), - trainable=True) - input_shape = np.delete(data_share.shape, 0, 0) - data1 = pfl_mpc.data(name='input', shape=input_shape, dtype='int64') - data2 = fluid.data(name='input1', shape=data_normal.shape, dtype='int64') - - math_embedding = fluid.input.embedding(input=data2, size=w_data.shape, param_attr=w_param_attrs1, dtype='float32') - - op_embedding = pfl_mpc.input.embedding(input=data1, size=(input_shape[1],input_shape[0]), param_attr=w_param_attrs, dtype='int64') - - exe = fluid.Executor(place=fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - - results = exe.run(feed={'input': data_share, 'input1': data_normal}, fetch_list=[op_embedding, math_embedding]) - - return_results.append(results[0]) - expected_result.append(results[1]) - - def test_embedding_op(self): - data = np.array([[1, 0, 0], [0, 1, 0]]) - data_normal = np.array([0, 1]).astype('int64') - w_data = np.array([[1, 2], [2, 3], [3, 4]]) - - # data = self.gen_one_hot(data_normal, w_data.shape[0]).astype('int64') - - data_share = aby3.make_shares(np.array(data)) - data_all3shares = np.array([aby3.get_aby3_shares(data_share, i) for i in range(3)]) - w_data_share = aby3.make_shares(w_data) - w_data_all3shares = np.array([aby3.get_aby3_shares(w_data_share, i) for i in range(3)]) - - return_results = Manager().list() - expect_results = Manager().list() - ret = self.multi_party_run(target=self.embedding_op, - data=data, - data_normal=data_normal, - w_data=w_data, - data_share=data_all3shares, - w_data_share=w_data_all3shares, - return_results=return_results, - expect_results=expect_results) - self.assertEqual(ret[0], True) - revealed = aby3.reconstruct(np.array(return_results)) - # print("reveal: ", revealed) - self.assertTrue(np.allclose(revealed, expect_results[0], atol=1e-4)) - - def test_mpc_one_hot(self): - data = np.array([0, 1]).astype('int64') - ret = self.gen_one_hot(data, 3) - mpc_one_hot = aby3.make_shares(ret) - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle_fl/mpc/tests/unittests/test_op_embedding.py b/python/paddle_fl/mpc/tests/unittests/test_op_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..4ef51022cd71f5de4fb539d320e16a7b694cd777 --- /dev/null +++ b/python/paddle_fl/mpc/tests/unittests/test_op_embedding.py @@ -0,0 +1,66 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module test embedding op. + +""" +import unittest +from multiprocessing import Manager +import numpy as np + +from op_test import OpTest +import paddle_fl.mpc.data_utils.aby3 as aby3 + +import paddle.fluid.core as core + +import paddle.fluid as fluid +from paddle.fluid import Program, program_guard + +class TestLookupTableOp(OpTest): + def to_one_hot(self, x, depth): + out = np.zeros(shape=(np.product(x.shape), depth)).astype('float') + for i in range(np.product(x.shape)): + out[i, x[i]] = 1.0 + return out + + def setUp(self): + OpTest.setUp(self) + self.op_type = "mpc_lookup_table_v2" + self.dtype = "int64" + table = np.random.random((17, 31)).astype("float") + ids = np.random.randint(0, 17, 4).astype("int64") + share = lambda x: np.array([x * 65536/3] * 2).astype('int64') + ids_one_hot = self.to_one_hot(ids, table.shape[0]) + mpc_table = share(table) + mpc_ids_one_hot = share(ids_one_hot) + self.inputs = {'W': mpc_table, 'Ids': mpc_ids_one_hot} + self.outputs = {'Out': table[ids]} + + def test_check_output(self): + place = core.CPUPlace() + self.check_output_with_place(place, atol=1e-3) + + def test_check_grad(self): + # set output type to 'int64' + # TODO: if not set outputs type to 'int64', exception will throw + self.outputs = {'Out': np.array([1]).astype('int64')} + place = core.CPUPlace() + self.check_grad_with_place(place, ['W'], 'Out', no_grad_set=set('Ids'), max_relative_error=0.01) + + + +if __name__ == "__main__": + unittest.main() + diff --git a/python/paddle_fl/mpc/tests/unittests/test_op_scale.py b/python/paddle_fl/mpc/tests/unittests/test_op_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..46cad5777cc15e4e1f1232b0be037a9d40ce85b0 --- /dev/null +++ b/python/paddle_fl/mpc/tests/unittests/test_op_scale.py @@ -0,0 +1,83 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module test scale op. + +""" +import unittest +from multiprocessing import Manager +import numpy as np + + +import test_op_base +from op_test import OpTest +import paddle_fl.mpc.data_utils.aby3 as aby3 + +import paddle.fluid as fluid +import paddle.fluid.core as core + + +class TestScaleOp(OpTest): + def setUp(self): + self.op_type = "mpc_scale" + self.dtype = np.int64 + self.init_dtype_type() + share = lambda x: np.array([x * 65536/3] * 2).astype('int64') + input_p = np.random.random((10, 10)) + self.inputs = {'X': share(input_p).astype(self.dtype)} + self.attrs = {'scale': -2.3} + self.outputs = { + 'Out': input_p * self.attrs['scale'] + } + + def init_dtype_type(self): + pass + + def test_check_output(self): + place = core.CPUPlace() + self.check_output_with_place(place, atol=1e-3,) + + def test_check_grad(self): + place = core.CPUPlace() + self.check_grad_with_place(place, ['X'], 'Out', max_relative_error=0.05) + + +class TestScaleOpScaleVariable(OpTest): + def setUp(self): + self.op_type = "mpc_scale" + self.dtype = np.int64 + self.init_dtype_type() + self.scale = -2.3 + share = lambda x: np.array([x * 65536/3] * 2).astype('int64') + input_p = np.random.random((10, 10)) + self.inputs = { + 'X': share(input_p), + 'ScaleTensor': np.array([self.scale]).astype('float') + } + self.attrs = {} + self.outputs = {'Out': input_p * self.scale} + + def init_dtype_type(self): + pass + + def test_check_output(self): + place = core.CPUPlace() + self.check_output_with_place(place, atol=1e-3) + + def test_check_grad(self): + place = core.CPUPlace() + self.check_grad_with_place(place, ['X'], 'Out', max_relative_error=0.05) + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle_fl/mpc/tests/unittests/testsuite.py b/python/paddle_fl/mpc/tests/unittests/testsuite.py index bd762710038dd6292cba3755b072e7f1aac92b4b..e601b242ff33a530ce4280e512d8286292a32032 100644 --- a/python/paddle_fl/mpc/tests/unittests/testsuite.py +++ b/python/paddle_fl/mpc/tests/unittests/testsuite.py @@ -153,26 +153,26 @@ def append_loss_ops(block, output_names): else: avg_sum = [] for cur_loss in mean_inputs: - cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[1]) + cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[2, 1]) op = block.append_op( inputs={"X": [cur_loss]}, outputs={"Out": [cur_avg_loss]}, - type="mean") + type="mpc_mean") op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) avg_sum.append(cur_avg_loss) - loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[1]) + loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[2, 1]) op_sum = block.append_op( - inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='sum') + inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='mpc_sum') op_sum.desc.infer_var_type(block.desc) op_sum.desc.infer_shape(block.desc) - loss = block.create_var(dtype=loss_sum.dtype, shape=[1]) + loss = block.create_var(dtype=loss_sum.dtype, shape=[2, 1]) op_loss = block.append_op( inputs={"X": loss_sum}, outputs={"Out": loss}, - type='scale', + type='mpc_scale', attrs={'scale': 1.0 / float(len(avg_sum))}) op_loss.desc.infer_var_type(block.desc) op_loss.desc.infer_shape(block.desc)