From 506092146d05aa94d2430c5b3dfb63d699dca858 Mon Sep 17 00:00:00 2001 From: ziyoujiyi <73728031+ziyoujiyi@users.noreply.github.com> Date: Wed, 12 Jan 2022 19:02:03 +0800 Subject: [PATCH] the_one_ps dirs reconstruct (#38804) * delete gloo connect retry * the_one_ps dirs reconstruct * . * . * create the_one_ps dirs * create the_one_ps dirs * create the_one_ps dirs * create the_one_ps dirs * create the_one_ps dirs * create the_one_ps dirs * the one ps dirs modify * the one ps dirs modify * the one ps dirs modify * the one ps dirs modify --- .../fluid/distributed/dataset_utils/README.md | 6 ++++++ paddle/fluid/distributed/ps/CMakeLists.txt | 0 paddle/fluid/distributed/ps/README.md | 3 +++ .../fluid/distributed/ps/coordinator/README.md | 3 +++ paddle/fluid/distributed/ps/ps.proto | 13 +++++++++++++ paddle/fluid/distributed/ps/service/README.md | 8 ++++++++ .../distributed/ps/wrapper/ps_cpu_wrapper.h | 18 ++++++++++++++++++ .../distributed/ps/wrapper/ps_gpu_wrapper.h | 18 ++++++++++++++++++ .../distributed/ps/wrapper/ps_heter_wrapper.h | 18 ++++++++++++++++++ .../fluid/distributed/ps/wrapper/ps_wrapper.h | 18 ++++++++++++++++++ python/paddle/distributed/ps/README.md | 3 +++ python/paddle/distributed/ps/__init__.py | 13 +++++++++++++ python/paddle/distributed/ps/the_one_ps.py | 13 +++++++++++++ .../ps/utils/compile_time_strategy.py | 13 +++++++++++++ .../paddle/distributed/ps/utils/ps_factory.py | 13 +++++++++++++ .../distributed/ps/utils/ps_infer_utils.py | 13 +++++++++++++ 16 files changed, 173 insertions(+) create mode 100755 paddle/fluid/distributed/dataset_utils/README.md create mode 100644 paddle/fluid/distributed/ps/CMakeLists.txt create mode 100755 paddle/fluid/distributed/ps/README.md create mode 100755 paddle/fluid/distributed/ps/coordinator/README.md create mode 100755 paddle/fluid/distributed/ps/ps.proto create mode 100755 paddle/fluid/distributed/ps/service/README.md create mode 100755 paddle/fluid/distributed/ps/wrapper/ps_cpu_wrapper.h create mode 100755 paddle/fluid/distributed/ps/wrapper/ps_gpu_wrapper.h create mode 100755 paddle/fluid/distributed/ps/wrapper/ps_heter_wrapper.h create mode 100755 paddle/fluid/distributed/ps/wrapper/ps_wrapper.h create mode 100755 python/paddle/distributed/ps/README.md create mode 100755 python/paddle/distributed/ps/__init__.py create mode 100755 python/paddle/distributed/ps/the_one_ps.py create mode 100755 python/paddle/distributed/ps/utils/compile_time_strategy.py create mode 100755 python/paddle/distributed/ps/utils/ps_factory.py create mode 100755 python/paddle/distributed/ps/utils/ps_infer_utils.py diff --git a/paddle/fluid/distributed/dataset_utils/README.md b/paddle/fluid/distributed/dataset_utils/README.md new file mode 100755 index 00000000000..b1637c185e6 --- /dev/null +++ b/paddle/fluid/distributed/dataset_utils/README.md @@ -0,0 +1,6 @@ +# 目录说明 + +> 干掉原来的 index_dataset 目录 +dataset 抽样工具类 +用户自定义数据处理so +流式dataserver相关类 diff --git a/paddle/fluid/distributed/ps/CMakeLists.txt b/paddle/fluid/distributed/ps/CMakeLists.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/paddle/fluid/distributed/ps/README.md b/paddle/fluid/distributed/ps/README.md new file mode 100755 index 00000000000..d287dcd1111 --- /dev/null +++ b/paddle/fluid/distributed/ps/README.md @@ -0,0 +1,3 @@ +# 目录说明 + +> 成型之后，上级目录的 table、thirdparty、table、service 目录可以删除，communicator_common.h 、fleet.cc、fleet.h 删除 diff --git a/paddle/fluid/distributed/ps/coordinator/README.md b/paddle/fluid/distributed/ps/coordinator/README.md new file mode 100755 index 00000000000..774c7a1809d --- /dev/null +++ b/paddle/fluid/distributed/ps/coordinator/README.md @@ -0,0 +1,3 @@ +# 目录说明 + +* Coordinator 功能类 diff --git a/paddle/fluid/distributed/ps/ps.proto b/paddle/fluid/distributed/ps/ps.proto new file mode 100755 index 00000000000..2691f637527 --- /dev/null +++ b/paddle/fluid/distributed/ps/ps.proto @@ -0,0 +1,13 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ \ No newline at end of file diff --git a/paddle/fluid/distributed/ps/service/README.md b/paddle/fluid/distributed/ps/service/README.md new file mode 100755 index 00000000000..a219e92c63b --- /dev/null +++ b/paddle/fluid/distributed/ps/service/README.md @@ -0,0 +1,8 @@ +# 目录说明 + +* PSServer +* PSClient +* PsService +* Communicator +* MessageBusFramework +* *.proto diff --git a/paddle/fluid/distributed/ps/wrapper/ps_cpu_wrapper.h b/paddle/fluid/distributed/ps/wrapper/ps_cpu_wrapper.h new file mode 100755 index 00000000000..2175bacb965 --- /dev/null +++ b/paddle/fluid/distributed/ps/wrapper/ps_cpu_wrapper.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_CPU_WRAPPER_H_ +#define PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_CPU_WRAPPER_H_ + +#endif // PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_CPU_WRAPPER_H_ diff --git a/paddle/fluid/distributed/ps/wrapper/ps_gpu_wrapper.h b/paddle/fluid/distributed/ps/wrapper/ps_gpu_wrapper.h new file mode 100755 index 00000000000..a73c74efff4 --- /dev/null +++ b/paddle/fluid/distributed/ps/wrapper/ps_gpu_wrapper.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_GPU_WRAPPER_H_ +#define PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_GPU_WRAPPER_H_ + +#endif // PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_GPU_WRAPPER_H_ diff --git a/paddle/fluid/distributed/ps/wrapper/ps_heter_wrapper.h b/paddle/fluid/distributed/ps/wrapper/ps_heter_wrapper.h new file mode 100755 index 00000000000..b1c15d1e203 --- /dev/null +++ b/paddle/fluid/distributed/ps/wrapper/ps_heter_wrapper.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_HETER_WRAPPER_H_ +#define PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_HETER_WRAPPER_H_ + +#endif // PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_HETER_WRAPPER_H_ diff --git a/paddle/fluid/distributed/ps/wrapper/ps_wrapper.h b/paddle/fluid/distributed/ps/wrapper/ps_wrapper.h new file mode 100755 index 00000000000..c92835aa995 --- /dev/null +++ b/paddle/fluid/distributed/ps/wrapper/ps_wrapper.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_WRAPPER_H_ +#define PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_WRAPPER_H_ + +#endif // PADDLE_FLUID_DISTRIBUTED_PS_WRAPPER_PS_WRAPPER_H_ diff --git a/python/paddle/distributed/ps/README.md b/python/paddle/distributed/ps/README.md new file mode 100755 index 00000000000..8d28031794f --- /dev/null +++ b/python/paddle/distributed/ps/README.md @@ -0,0 +1,3 @@ +# 目录说明 + +* 改完之后，上层目录中 fleet 中相关文件（夹）就可以删除 diff --git a/python/paddle/distributed/ps/__init__.py b/python/paddle/distributed/ps/__init__.py new file mode 100755 index 00000000000..97043fd7ba6 --- /dev/null +++ b/python/paddle/distributed/ps/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/distributed/ps/the_one_ps.py b/python/paddle/distributed/ps/the_one_ps.py new file mode 100755 index 00000000000..e1663029ef1 --- /dev/null +++ b/python/paddle/distributed/ps/the_one_ps.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/distributed/ps/utils/compile_time_strategy.py b/python/paddle/distributed/ps/utils/compile_time_strategy.py new file mode 100755 index 00000000000..e1663029ef1 --- /dev/null +++ b/python/paddle/distributed/ps/utils/compile_time_strategy.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/distributed/ps/utils/ps_factory.py b/python/paddle/distributed/ps/utils/ps_factory.py new file mode 100755 index 00000000000..e1663029ef1 --- /dev/null +++ b/python/paddle/distributed/ps/utils/ps_factory.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/paddle/distributed/ps/utils/ps_infer_utils.py b/python/paddle/distributed/ps/utils/ps_infer_utils.py new file mode 100755 index 00000000000..e1663029ef1 --- /dev/null +++ b/python/paddle/distributed/ps/utils/ps_infer_utils.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -- GitLab