Skip to content

P
Paddle

PaddlePaddle / Paddle
1 年多 前同步成功

通知 2302
Star 20931
Fork 5422
P
Paddle
未验证 提交 09ae2852 编写于 作者: K kangguangli 提交者： GitHub
浏览文件
操作

remove unit tests about GraphExecutionOptimizer (#51575) 

* remove unit tests about GraphExecutionOptimizer

* remove test file
上级 521bba9c
隐藏空白更改
内联 并排
Showing with 0 addition and 472 deletion
python/paddle/fluid/tests/unittests/CMakeLists.txt
浏览文件 @ 09ae2852
......@@ -520,7 +520,6 @@ endforeach()
if((NOT WITH_GPU)
AND (NOT WITH_XPU)
AND NOT (WITH_ASCEND OR WITH_ASCEND_CL))
list(REMOVE_ITEM TEST_OPS "test_fleet_graph_execution_meta_optimizer")
list(REMOVE_ITEM TEST_OPS "test_dist_mnist_batch_merge")
endif()
......
python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt
浏览文件 @ 09ae2852
......@@ -138,22 +138,6 @@ if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
set_tests_properties(test_dygraph_sharding_stage3_for_eager PROPERTIES TIMEOUT
"350")
endif()
if((WITH_GPU
OR WITH_XPU
OR WITH_ASCEND
OR WITH_ASCEND_CL
)
AND LOCAL_ALL_PLAT)
bash_test_modules(
test_fleet_graph_execution_meta_optimizer
START_BASH
../../dist_test.sh
LABELS
"RUN_TYPE=DIST"
ENVS
"PADDLE_DIST_UT_PORT=21216;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
)
endif()
if(WITH_NCCL)
if(LOCAL_ALL_ARCH AND LOCAL_ALL_PLAT)
py_test_modules(
......@@ -167,21 +151,6 @@ if(WITH_NCCL)
PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=DIST")
endif()
endif()
if((WITH_GPU
OR WITH_XPU
OR WITH_ASCEND
OR WITH_ASCEND_CL
)
AND LOCAL_ALL_PLAT)
bash_test_modules(
test_fleet_graph_executor
START_BASH
../../dist_test.sh
LABELS
"RUN_TYPE=DIST"
ENVS
"http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
endif()
if((WITH_GPU) AND LOCAL_ALL_PLAT)
bash_test_modules(
test_parallel_dygraph_pipeline_parallel
......
python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py 已删除 100644 → 0
浏览文件 @ 521bba9c
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
from launch_function_helper import _find_free_port, launch_func, wait
class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase):
def setUp(self):
try:
self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"])
self._dist_ut_port_1 = self._dist_ut_port_0 + 1
except Exception as e:
self._dist_ut_port_0 = _find_free_port(set())
self._dist_ut_port_1 = _find_free_port(set())
def test_graph_execution_optimizer_not_apply(self):
port_a = self._dist_ut_port_0
port_b = self._dist_ut_port_1
node_a = {
"PADDLE_TRAINER_ID": "0",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "0",
}
node_b = {
"PADDLE_TRAINER_ID": "1",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "1",
}
def node_func():
import paddle
paddle.enable_static()
import paddle.distributed.fleet as fleet
fleet.init(is_collective=True)
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32'
)
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = paddle.static.nn.fc(x=input_x, size=64, activation='tanh')
fc_2 = paddle.static.nn.fc(x=fc_1, size=64, activation='tanh')
prediction = paddle.static.nn.fc(
x=[fc_2], size=2, activation='softmax'
)
cost = paddle.nn.functional.cross_entropy(
input=prediction,
label=input_y,
reduction='none',
use_softmax=False,
)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy
)
optimizer.minimize(avg_cost)
exe = paddle.fluid.Executor()
exe.run(paddle.fluid.default_startup_program())
proc_a = launch_func(node_func, node_a)
proc_a.start()
proc_b = launch_func(node_func, node_b)
proc_b.start()
wait([proc_a, proc_b])
def test_graph_execution_optimizer(self):
port_a = self._dist_ut_port_0 + 2
port_b = self._dist_ut_port_1 + 2
node_a = {
"PADDLE_TRAINER_ID": "0",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "0",
}
node_b = {
"PADDLE_TRAINER_ID": "1",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "1",
}
def node_func():
import paddle
paddle.enable_static()
import paddle.distributed.fleet as fleet
fleet.init(is_collective=True)
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32'
)
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = paddle.static.nn.fc(x=input_x, size=64, activation='tanh')
fc_2 = paddle.static.nn.fc(x=fc_1, size=64, activation='tanh')
prediction = paddle.static.nn.fc(
x=[fc_2], size=2, activation='softmax'
)
cost = paddle.nn.functional.cross_entropy(
input=prediction,
label=input_y,
reduction='none',
use_softmax=False,
)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.nccl_comm_num = 2
strategy.sync_nccl_allreduce = True
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy
)
optimizer.minimize(avg_cost)
exe = paddle.fluid.Executor()
exe.run(paddle.fluid.default_startup_program())
import numpy as np
def gen_data():
return {
"x": np.random.random(size=(128, 32)).astype('float32'),
"y": np.random.randint(2, size=(128, 1)).astype('int64'),
}
for i in range(10):
cost_val = exe.run(feed=gen_data(), fetch_list=[avg_cost.name])
print("cost of step[{}] = {}".format(i, cost_val))
proc_a = launch_func(node_func, node_a)
proc_a.start()
proc_b = launch_func(node_func, node_b)
proc_b.start()
wait([proc_a, proc_b])
def test_graph_execution_optimizer_not_apply_v2(self):
port_a = self._dist_ut_port_0 + 4
port_b = self._dist_ut_port_1 + 4
node_a = {
"PADDLE_TRAINER_ID": "0",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "0",
}
node_b = {
"PADDLE_TRAINER_ID": "1",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "1",
}
def node_func():
import paddle
paddle.enable_static()
import paddle.distributed.fleet as fleet
fleet.init(is_collective=True)
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32'
)
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = paddle.static.nn.fc(x=input_x, size=64, activation='tanh')
fc_2 = paddle.static.nn.fc(x=fc_1, size=64, activation='tanh')
prediction = paddle.static.nn.fc(
x=[fc_2], size=2, activation='softmax'
)
cost = paddle.nn.functional.cross_entropy(
input=prediction,
label=input_y,
reduction='none',
use_softmax=False,
)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy
)
optimizer.minimize(avg_cost)
exe = paddle.fluid.Executor()
exe.run(paddle.fluid.default_startup_program())
proc_a = launch_func(node_func, node_a)
proc_a.start()
proc_b = launch_func(node_func, node_b)
proc_b.start()
wait([proc_a, proc_b])
def test_graph_execution_optimizer_v2(self):
port_a = self._dist_ut_port_0 + 6
port_b = self._dist_ut_port_1 + 6
node_a = {
"PADDLE_TRAINER_ID": "0",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "0",
}
node_b = {
"PADDLE_TRAINER_ID": "1",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b),
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format(
port_a, port_b
),
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "1",
}
def node_func():
import paddle
paddle.enable_static()
import paddle.distributed.fleet as fleet
fleet.init(is_collective=True)
input_x = paddle.static.data(
name="x", shape=[-1, 32], dtype='float32'
)
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = paddle.static.nn.fc(x=input_x, size=64, activation='tanh')
fc_2 = paddle.static.nn.fc(x=fc_1, size=64, activation='tanh')
prediction = paddle.static.nn.fc(
x=[fc_2], size=2, activation='softmax'
)
cost = paddle.nn.functional.cross_entropy(
input=prediction,
label=input_y,
reduction='none',
use_softmax=False,
)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.nccl_comm_num = 2
strategy.sync_nccl_allreduce = True
optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(
optimizer, strategy=strategy
)
optimizer.minimize(avg_cost)
exe = paddle.fluid.Executor()
exe.run(paddle.fluid.default_startup_program())
import numpy as np
def gen_data():
return {
"x": np.random.random(size=(128, 32)).astype('float32'),
"y": np.random.randint(2, size=(128, 1)).astype('int64'),
}
for i in range(10):
cost_val = exe.run(feed=gen_data(), fetch_list=[avg_cost.name])
print("cost of step[{}] = {}".format(i, cost_val))
proc_a = launch_func(node_func, node_a)
proc_a.start()
proc_b = launch_func(node_func, node_b)
proc_b.start()
wait([proc_a, proc_b])
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py 已删除 100644 → 0
浏览文件 @ 521bba9c
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
from launch_function_helper import launch_func
def node_func():
import paddle
import paddle.distributed.fleet as fleet
import paddle.distributed.fleet.base.role_maker as role_maker
paddle.enable_static()
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
input_x = paddle.static.data(name="x", shape=[-1, 32], dtype='float32')
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = paddle.static.nn.fc(x=input_x, size=64, activation='tanh')
fc_2 = paddle.static.nn.fc(x=fc_1, size=64, activation='tanh')
prediction = paddle.static.nn.fc(x=[fc_2], size=2, activation='softmax')
cost = paddle.nn.functional.cross_entropy(
input=prediction,
label=input_y,
reduction='none',
use_softmax=False,
)
avg_cost = paddle.mean(x=cost)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.nccl_comm_num = 2
strategy.sync_nccl_allreduce = True
optimizer = paddle.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
exe = paddle.fluid.Executor()
exe.run(paddle.fluid.default_startup_program())
import numpy as np
def gen_data():
return {
"x": np.random.random(size=(128, 32)).astype('float32'),
"y": np.random.randint(2, size=(128, 1)).astype('int64'),
}
for i in range(5):
cost_val = exe.run(feed=gen_data(), fetch_list=[avg_cost.name])
print("cost of step[{}] = {}".format(i, cost_val))
class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase):
def test_graph_execution_optimizer(self):
node_a = {
"PADDLE_TRAINER_ID": "0",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:36001",
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002",
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "0",
}
node_b = {
"PADDLE_TRAINER_ID": "1",
"PADDLE_CURRENT_ENDPOINT": "127.0.0.1:36002",
"PADDLE_TRAINERS_NUM": "2",
"PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002",
"http_proxy": "",
"https_proxy": "",
"FLAGS_selected_gpus": "1",
}
# rank 1
proc_b = launch_func(node_func, node_b)
proc_b.start()
# rank 0, for wait server ready coverage
# just for coverage
for key in node_a:
os.environ[key] = node_a[key]
node_func()
proc_b.join()
if __name__ == "__main__":
unittest.main()
python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv
浏览文件 @ 09ae2852
......@@ -11,9 +11,7 @@ test_rnn_dp,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;ht
test_parallel_dygraph_mp_layers,,GPU,120,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
test_tcp_store,LINUX;APPLE,,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_dygraph_sharding_stage3_for_eager,,,350,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_graph_execution_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_communicator_half_async,,,120,DIST,test_runner.py,2,,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
test_fleet_graph_executor,,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_parallel_dygraph_pipeline_parallel,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_parallel_dygraph_pipeline_parallel_with_virtual_stage,,GPU,500,DIST,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
test_fleet_localsgd_meta_optimizer,LINUX,GPU;XPU;ASCEND;ASCEND_CL,,,test_runner.py,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
......
tools/parallel_UT_rule.py
浏览文件 @ 09ae2852
......@@ -265,7 +265,6 @@ HIGH_PARALLEL_JOB_NEW = [
'test_pool2d_int8_mkldnn_op',
'test_mul_int8_mkldnn_op',
'test_scale_matmul_fuse_pass',
'test_fleet_graph_executor',
'decorator_test',
'test_collective_base',
'test_multi_gru_mkldnn_op',
......@@ -536,7 +535,6 @@ HIGH_PARALLEL_JOB_NEW = [
'test_dist_sparse_tensor_load_rmsprop',
'test_collective_split_embedding_none_divisible',
'test_parallel_dygraph_dataparallel',
'test_fleet_graph_execution_meta_optimizer',
'test_dist_fleet_ps3',
'test_dist_mnist_pg',
'test_pipeline_parallel',
......@@ -2101,7 +2099,6 @@ CPU_PARALLEL_JOB = [
'test_auto_checkpoint',
'test_fleet_pipeline_meta_optimizer',
'test_dist_fleet_heter_ctr',
'test_fleet_graph_execution_meta_optimizer',
'test_fleet_run_random_port',
'test_dist_fleet_ps5',
'test_dist_fleet_a_sync_optimizer_auto',
......@@ -2175,7 +2172,6 @@ CPU_PARALLEL_JOB = [
'test_fleet_meta_optimizer_base',
'table_test',
'test_fleet_rolemaker_new',
'test_fleet_graph_executor',
'test_multi_out_jit',
'test_fleet_utils',
'brpc_service_dense_sgd_test',
......
tools/static_mode_white_list.py
浏览文件 @ 09ae2852
......@@ -649,7 +649,6 @@ STATIC_MODE_TESTING_LIST = [
'test_mix_precision_all_reduce_fuse',
'test_rank_attention_op',
'test_fleet_base',
'test_fleet_graph_executor',
'test_fleet_meta_optimizer_base',
'test_ir_memory_optimize_transformer',
'test_trt_fc_fuse_pass',
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或