diff --git a/python/paddle_fl/mpc/data_utils/alignment.py b/python/paddle_fl/mpc/data_utils/alignment.py index 8ee9978fd0c5cfe1fe16945df5e13caf92017b40..0e8dab74853c6e596a50b7eec3b6eb2256fc6cd3 100644 --- a/python/paddle_fl/mpc/data_utils/alignment.py +++ b/python/paddle_fl/mpc/data_utils/alignment.py @@ -18,7 +18,7 @@ PSI (Private Set Intersection) algorithm. import os import sys import mpc_data_utils as mdu - +from multiprocessing.connection import Client, Listener __all__ = ['align', ] @@ -90,7 +90,6 @@ def _send_align_result(result, send_list): Each party is represented as "id:ip:port". :return: """ - from multiprocessing.connection import Client for host in send_list: ip_addr = host.split(":")[1] port = int(host.split(":")[2]) @@ -112,4 +111,7 @@ def _recv_align_result(host): port = int(host.split(":")[2]) server = Listener((ip_addr, port)) conn = server.accept() - return conn.recv() + result = conn.recv() + conn.close() + server.close() + return result diff --git a/python/paddle_fl/mpc/examples/env_set.py b/python/paddle_fl/mpc/examples/env_set.py deleted file mode 100644 index 86404094853083b57a757fa79c71ef66e228e18d..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/env_set.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -setup proper fluid_encrypted pkg path -""" -import sys - - -class TestOptions: - """ - Read test options of role, server ip, and port - """ - - def __init__(self): - if len(sys.argv) < 4: - print("Args: $ROLE $SERVER $PORT") - sys.exit() - self.__role = sys.argv[1] - self.__server = sys.argv[2] - self.__port = sys.argv[3] - - def values(self): - """ - Return argument values - """ - return self.__role, self.__server, self.__port diff --git a/python/paddle_fl/mpc/examples/mnist_demo/mnist_demo.py b/python/paddle_fl/mpc/examples/mnist_demo/mnist_demo.py index efa69836b13ad996f19967980da6d4c9543cf181..5d7c5461bade14385f4da615cc3eb2fa44d5153e 100644 --- a/python/paddle_fl/mpc/examples/mnist_demo/mnist_demo.py +++ b/python/paddle_fl/mpc/examples/mnist_demo/mnist_demo.py @@ -1,4 +1,3 @@ -# coding=utf-8 # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +16,7 @@ MNIST Demo """ import sys -sys.path.append('../') -import env_set import numpy as np import time @@ -29,7 +26,7 @@ import paddle_fl.mpc as pfl_mpc import paddle_fl.mpc.data_utils.aby3 as aby3 import prepare_data -role, server, port = env_set.TestOptions().values() +role, server, port = sys.argv[1], sys.argv[2], sys.argv[3] # modify host(localhost). pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) role = int(role) diff --git a/python/paddle_fl/mpc/examples/mnist_demo/run_standalone.sh b/python/paddle_fl/mpc/examples/mnist_demo/run_standalone.sh index 4c6041fbad34663f8e888bad8eb27c9e406aed63..244f37a6472f8705f29a72302bf632d731ac97ea 100755 --- a/python/paddle_fl/mpc/examples/mnist_demo/run_standalone.sh +++ b/python/paddle_fl/mpc/examples/mnist_demo/run_standalone.sh @@ -61,10 +61,23 @@ fi # clear the redis cache $REDIS_BIN -h $SERVER -p $PORT flushall +# remove temp data generated in last time +LOSS_FILE="/tmp/mnist_output_loss.*" +PRED_FILE="/tmp/mnist_output_prediction.*" +if [ "$LOSS_FILE" ]; then + rm -rf $LOSS_FILE +fi + +if [ "$PRED_FILE" ]; then + rm -rf $PRED_FILE +fi + +# kick off script with roles of 1 and 2, and redirect output to /dev/null for role in {1..2}; do $PYTHON $SCRIPT $role $SERVER $PORT 2>&1 >/dev/null & done # for party of role 0, run in a foreground mode and show the output $PYTHON $SCRIPT 0 $SERVER $PORT + diff --git a/python/paddle_fl/mpc/examples/run_standalone.sh b/python/paddle_fl/mpc/examples/run_standalone.sh deleted file mode 100755 index 4c6041fbad34663f8e888bad8eb27c9e406aed63..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/run_standalone.sh +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/bin/bash -# -# A tools to faciliate the parallel running of fluid_encrypted test scrips. -# A test script is EXPECTED to accepted arguments in the following format: -# -# SCRIPT_NAME $ROLE $SERVER $PORT -# ROLE: the role of the running party -# SERVER: the address of the party discovering service -# PORT: the port of the party discovering service -# -# This tool will try to fill the above three argument to the test script, -# so that totally three processes running the script will be started, to -# simulate run of three party in a standalone machine. -# -# Usage of this script: -# -# bash run_standalone.sh TEST_SCRIPT_NAME -# - -# modify the following vars according to your environment -PYTHON="python" -REDIS_HOME="path_to_redis_bin" -SERVER="localhost" -PORT=9937 - -function usage() { - echo 'run_standalone.sh SCRIPT_NAME [ARG...]' - exit 0 -} - -if [ $# -lt 1 ]; then - usage -fi - -SCRIPT=$1 -if [ ! -f $SCRIPT ]; then - echo 'Could not find script of '$SCRIPT - exit 1 -fi - -REDIS_BIN=$REDIS_HOME/redis-cli -if [ ! -f $REDIS_BIN ]; then - echo 'Could not find redis cli in '$REDIS_HOME - exit 1 -fi - -# clear the redis cache -$REDIS_BIN -h $SERVER -p $PORT flushall - - -for role in {1..2}; do - $PYTHON $SCRIPT $role $SERVER $PORT 2>&1 >/dev/null & -done - -# for party of role 0, run in a foreground mode and show the output -$PYTHON $SCRIPT 0 $SERVER $PORT diff --git a/python/paddle_fl/mpc/examples/test_add.py b/python/paddle_fl/mpc/examples/test_add.py deleted file mode 100644 index 05895b3a832912e4b5bfaa0c77eb5f68b1854ad7..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_add.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -test mpc add op -""" - -# set proper path for fluid_encrypted without install, should be first line -import env_set - -import sys -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -# call mpc add -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) - -data_1 = pfl_mpc.data(name='data_1', shape=[8], dtype='int64') -data_2 = pfl_mpc.data(name='data_2', shape=[8], dtype='int64') - -d_1 = np.array( - [[0, 1, 2, 3, 4, 5, 6, 7], [0, 1, 2, 3, 4, 5, 6, 7]]).astype('int64') -d_2 = np.array( - [[7, 6, 5, 4, 3, 2, 1, 0], [7, 6, 5, 4, 3, 2, 1, 0]]).astype('int64') - -out_add = data_1 + data_2 - -exe = fluid.Executor(place=fluid.CPUPlace()) -out_add = exe.run(feed={ - 'data_1': d_1, - 'data_2': d_2, -}, fetch_list=[out_add]) - -print(out_add) diff --git a/python/paddle_fl/mpc/examples/test_compare.py b/python/paddle_fl/mpc/examples/test_compare.py deleted file mode 100644 index ab2991767c80fde49c815e437d85a5992fb015b4..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_compare.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -test mixed mpc ops -""" - -# set proper path for fluid_encrypted without install, should be first line -import env_set - -import time -import sys -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) - -data_1 = pfl_mpc.data(name='data_1', shape=[2, 2], dtype='int64') -data_2 = fluid.data(name='data_2', shape=[1, 2, 2], dtype='float32') - -out_gt = data_1 > data_2 -out_ge = data_1 >= data_2 -out_lt = data_1 < data_2 -out_le = data_1 <= data_2 -out_eq = data_1 == data_2 -out_neq = data_1 != data_2 - -d_1 = np.array([[[65536, 65536], [65536, 65536]], - [[65536, 65536], [65536, 65536]]]).astype('int64') -d_2 = np.array([[[10, 3], [0, -3]]]).astype('float32') - -exe = fluid.Executor(place=fluid.CPUPlace()) -exe.run(fluid.default_startup_program()) - -out_gt, out_ge, out_lt, out_le, out_eq, out_neq = exe.run( - feed={'data_1': d_1, - 'data_2': d_2}, - fetch_list=[out_gt, out_ge, out_lt, out_le, out_eq, out_neq]) - -print("Input: \n d_1: {}\n d_2: {}\n".format(d_1, d_2)) - -print( - "greater_than:{}\n greater_equal:{} \n less_than:{} \n less_equal:{} \n equal:{} \n not_equal:{} \n" - .format(out_gt, out_ge, out_lt, out_le, out_eq, out_neq)) diff --git a/python/paddle_fl/mpc/examples/test_data_utils.py b/python/paddle_fl/mpc/examples/test_data_utils.py deleted file mode 100644 index 0b5d466310776a20776c5dbc6ffcbb6ed0a6150a..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_data_utils.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# export -import sys - -import data_utils as du -import numpy as np - -plaintext = 2.9 -print("Plaintext: {0}".format(plaintext)) - -shares = du.share(plaintext) -print("Shares: {0}".format(shares)) - -revealed_text = du.reveal(shares) -print("Revealed plaintext: {0}".format(revealed_text)) diff --git a/python/paddle_fl/mpc/examples/test_init.py b/python/paddle_fl/mpc/examples/test_init.py deleted file mode 100644 index 837ef11494635e7aa992c111be7d09fbef790dc2..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_init.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -test mpc init op -""" - -# set proper path for fluid_encrypted without install, should be first line -import env_set - -import sys -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -# call mpc init -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) -exe = fluid.Executor(place=fluid.CPUPlace()) -exe.run() - -print("Role " + role + " started") diff --git a/python/paddle_fl/mpc/examples/test_mixed.py b/python/paddle_fl/mpc/examples/test_mixed.py deleted file mode 100644 index ef917fd36b0096d46af384025bd6f70268e2c9f8..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_mixed.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -test mixed mpc ops -""" - -# set proper path for fluid_encrypted without install, should be first line -import env_set - -import time -import sys -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -# call mpc add -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) - -data_1 = pfl_mpc.data(name='data_1', shape=[2, 2], dtype='int64') -data_2 = pfl_mpc.data(name='data_2', shape=[2, 2], dtype='int64') -data_3 = fluid.data(name='data_3', shape=[1, 2, 2], dtype='int64') - -out_sub = data_1 - data_2 -out_mul = pfl_mpc.layers.mul(x=data_1, y=data_2) -out_mean = pfl_mpc.layers.mean(x=data_1) -out_square = pfl_mpc.layers.square(x=data_1) -out_sum = pfl_mpc.layers.sum([data_1, data_2]) - -d_1 = np.array([[[10, 10], [10, 10]], [[10, 10], [10, 10]]]).astype('int64') -d_2 = np.array([[[5, 5], [5, 5]], [[5, 5], [5, 5]]]).astype('int64') - -exe = fluid.Executor(place=fluid.CPUPlace()) -exe.run(fluid.default_startup_program()) - -out_sub, out_mul, out_mean, out_square, out_sum = exe.run( - feed={'data_1': d_1, - 'data_2': d_2}, - fetch_list=[out_sub, out_mul, out_mean, out_square, out_sum]) -print("sub:{},\n mul: {},\n mean:{},\n square:{}, \n sum: {}\n" - .format(out_sub, out_mul, out_mean, out_square, out_sum)) diff --git a/python/paddle_fl/mpc/examples/test_ml.py b/python/paddle_fl/mpc/examples/test_ml.py deleted file mode 100644 index c60add9fd669e437fc7ad51cd3906890b61aaa6b..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_ml.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -test mixed mpc ops -""" - -# set proper path for fluid_encrypted without install, should be first line -import env_set - -import time -import sys -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) - -data_1 = pfl_mpc.data(name='data_1', shape=[2, 2], dtype='int64') -data_2 = pfl_mpc.data(name='data_2', shape=[2, 2], dtype='int64') - -out_sec = pfl_mpc.layers.square_error_cost(input=data_1, label=data_2) -out_fc = pfl_mpc.layers.fc(input=data_1, size=1, act=None) -out_relu = pfl_mpc.layers.relu(data_1) - -d_1 = np.array([[[10, 10], [10, 10]], [[10, 10], [10, 10]]]).astype('int64') -d_2 = np.array([[[5, 5], [5, 5]], [[5, 5], [5, 5]]]).astype('int64') - -exe = fluid.Executor(place=fluid.CPUPlace()) -exe.run(fluid.default_startup_program()) - -out_sec, out_fc, out_relu = exe.run(feed={'data_1': d_1, - 'data_2': d_2}, - fetch_list=[out_sec, out_fc, out_relu]) -print("square_error_cost: {},\n out_fc:{},\n out_relu:{}\n".format( - out_sec, out_fc, out_relu)) diff --git a/python/paddle_fl/mpc/examples/test_net.py b/python/paddle_fl/mpc/examples/test_net.py deleted file mode 100644 index 37043929fa75969346f6ba8632fe794119a80426..0000000000000000000000000000000000000000 --- a/python/paddle_fl/mpc/examples/test_net.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import env_set -import numpy as np -import paddle.fluid as fluid -import paddle_fl.mpc as pfl_mpc - -role, server, port = env_set.TestOptions().values() - -pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) - -batch_size = 3 - -# x is in cypher text type -x = pfl_mpc.data(name='x', shape=[batch_size, 8], dtype='int64') -# y is in cypher text type -y = pfl_mpc.data(name='y', shape=[batch_size, 1], dtype='int64') - -y_pre = pfl_mpc.layers.fc(input=x, size=1, act=None) -y_relu = pfl_mpc.layers.relu(input=y_pre) -cost = pfl_mpc.layers.square_error_cost(input=y_relu, label=y) -avg_loss = pfl_mpc.layers.mean(cost) - -optimizer = pfl_mpc.optimizer.SGD(learning_rate=0.001) -optimizer.minimize(avg_loss) - -exe = fluid.Executor(place=fluid.CPUPlace()) -exe.run(fluid.default_startup_program()) - -iters = 1 -for _ in range(iters): - d_1 = np.ones(shape=(2, batch_size, 8), dtype='int64') - d_2 = np.zeros(shape=(2, batch_size, 1), dtype='int64') - loss = exe.run(feed={'x': d_1, 'y': d_2}, fetch_list=[avg_loss]) - print(loss) diff --git a/python/paddle_fl/mpc/examples/uci_demo/run_standalone.sh b/python/paddle_fl/mpc/examples/uci_demo/run_standalone.sh index 4c6041fbad34663f8e888bad8eb27c9e406aed63..ce5993e086e028021ada5b5942efde9432b55305 100755 --- a/python/paddle_fl/mpc/examples/uci_demo/run_standalone.sh +++ b/python/paddle_fl/mpc/examples/uci_demo/run_standalone.sh @@ -61,10 +61,23 @@ fi # clear the redis cache $REDIS_BIN -h $SERVER -p $PORT flushall +# remove temp data generated in last time +LOSS_FILE="/tmp/uci_loss.*" +PRED_FILE="/tmp/uci_prediction.*" +if [ "$LOSS_FILE" ]; then + rm -rf $LOSS_FILE +fi + +if [ "$PRED_FILE" ]; then + rm -rf $PRED_FILE +fi + +# kick off script with roles of 1 and 2, and redirect output to /dev/null for role in {1..2}; do $PYTHON $SCRIPT $role $SERVER $PORT 2>&1 >/dev/null & done # for party of role 0, run in a foreground mode and show the output $PYTHON $SCRIPT 0 $SERVER $PORT + diff --git a/python/paddle_fl/mpc/examples/uci_demo/uci_housing_demo.py b/python/paddle_fl/mpc/examples/uci_demo/uci_housing_demo.py index 7d41bd48638f42ab672b2a11aba42718488cc226..a14f01573d012ea101bf9402d0c60223b59d9ffe 100644 --- a/python/paddle_fl/mpc/examples/uci_demo/uci_housing_demo.py +++ b/python/paddle_fl/mpc/examples/uci_demo/uci_housing_demo.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys -sys.path.append('../') - -import env_set import numpy as np import time @@ -23,7 +20,7 @@ import paddle.fluid as fluid import paddle_fl.mpc as pfl_mpc import paddle_fl.mpc.data_utils.aby3 as aby3 -role, server, port = env_set.TestOptions().values() +role, server, port = sys.argv[1], sys.argv[2], sys.argv[3] pfl_mpc.init("aby3", int(role), "localhost", server, int(port)) role = int(role) diff --git a/python/setup.py b/python/setup.py index 1dc34c07e2c22405971aaa628fa1088667660b83..c03655a4c2445b2c9a7e0f1b65d75d7d07a5bad0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -34,7 +34,7 @@ def python_version(): max_version, mid_version, min_version = python_version() REQUIRED_PACKAGES = [ - 'six >= 1.10.0', 'protobuf >= 3.1.0', 'paddlepaddle == 1.8.0', 'paddlepaddle-gpu >= 1.8' + 'six >= 1.10.0', 'protobuf >= 3.1.0', 'paddlepaddle >= 1.8.0', 'paddlepaddle-gpu >= 1.8' ] if max_version < 3: