mv ps distributed dir (#52885)

* mv ps distributed dir

* fix

* add del auto_parallel

* add auto_parallel

* fix ps

* fix bug

* fix test bug

* fix test bug

* merge develop fix error

* merge develop fix error

* merge develop fix error

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -560,9 +560,6 @@ set_tests_properties(test_norm_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 
 if(WITH_DISTRIBUTE)
-  add_subdirectory(distributed_passes)
-  add_subdirectory(ps)
-  add_subdirectory(auto_parallel)
   add_subdirectory(collective)
 
   # FIXME(typhoonzero): add these tests back
@@ -769,7 +766,6 @@ if(NOT WIN32)
 endif()
 
 add_subdirectory(sequence)
-add_subdirectory(distribution)
 
 # dist xpu tests:
 if(WITH_XPU_BKCL)

test/CMakeLists.txt

@@ -93,10 +93,8 @@ endfunction()
 if(WITH_TESTING)
   add_subdirectory(amp)
   add_subdirectory(asp)
-  # add_subdirectory(auto_parallel)
   add_subdirectory(autograd)
   add_subdirectory(book)
-  add_subdirectory(collective)
   # add_subdirectory(composite_ops)
   add_subdirectory(contrib)
   add_subdirectory(cpp)
@@ -104,8 +102,13 @@ if(WITH_TESTING)
   add_subdirectory(custom_op)
   add_subdirectory(custom_runtime)
   add_subdirectory(dataset)
-  # add_subdirectory(distributed_passes)
-  # add_subdirectory(distribution)
+  if(WITH_DISTRIBUTE)
+    add_subdirectory(collective)
+    add_subdirectory(auto_parallel)
+    add_subdirectory(distributed_passes)
+    add_subdirectory(ps)
+  endif()
+  add_subdirectory(distribution)
   add_subdirectory(dygraph_to_static)
   if(NOT WIN32 OR NOT WITH_GPU)
     add_subdirectory(fft)
@@ -120,7 +123,6 @@ if(WITH_TESTING)
     add_subdirectory(mkldnn)
   endif()
   add_subdirectory(prim)
-  # add_subdirectory(ps)
   add_subdirectory(quantization)
   add_subdirectory(rnn)
   add_subdirectory(rpc)

python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py → test/auto_parallel/get_gpt_model.py

@@ -20,7 +20,7 @@ import numpy as np
 import paddle
 from paddle.distributed.fleet import auto
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_fused_linear_pass.py → test/auto_parallel/test_fused_linear_pass.py

@@ -22,7 +22,7 @@ from get_gpt_model import FakeDataset, generate_model
 import paddle
 from paddle.distributed.fleet import auto
 
-sys.path.append("..")
+sys.path.append("../../python/paddle/fluid/tests/unittests")
 from test_sparse_addmm_op import get_cuda_version
 
 

python/paddle/fluid/tests/unittests/auto_parallel/test_group_operators.py → test/auto_parallel/test_group_operators.py

@@ -20,7 +20,7 @@ import numpy as np
 import paddle
 from paddle import static
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py → test/auto_parallel/test_parallel_tuner.py

@@ -26,7 +26,7 @@ from paddle.distributed.auto_parallel.dist_context import (
 from paddle.distributed.auto_parallel.process_mesh import ProcessMesh
 from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py → test/auto_parallel/test_parallel_tuner_full.py

@@ -28,7 +28,7 @@ from paddle.distributed.auto_parallel.process_mesh import ProcessMesh
 from paddle.distributed.auto_parallel.strategy import Strategy
 from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py → test/auto_parallel/test_parallel_tuner_predict.py

@@ -26,7 +26,7 @@ from paddle.distributed.auto_parallel.dist_context import (
 from paddle.distributed.auto_parallel.process_mesh import ProcessMesh
 from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_pass_base_list.py → test/auto_parallel/test_pass_base_list.py

@@ -22,7 +22,7 @@ from get_gpt_model import FakeDataset, generate_model
 import paddle
 from paddle.distributed.fleet import auto
 
-sys.path.append("..")
+sys.path.append("../../python/paddle/fluid/tests/unittests")
 from test_sparse_addmm_op import get_cuda_version
 
 

python/paddle/fluid/tests/unittests/auto_parallel/test_pattern.py → test/auto_parallel/test_pattern.py

@@ -20,7 +20,7 @@ import numpy as np
 import paddle
 from paddle import static
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_pattern_match.py → test/auto_parallel/test_pattern_match.py

@@ -20,7 +20,7 @@ import numpy as np
 import paddle
 from paddle import static
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/auto_parallel/test_rule_based_tuner.py → test/auto_parallel/test_rule_based_tuner.py

@@ -20,7 +20,7 @@ import numpy as np
 import paddle
 from paddle import static
 
-sys.path.append("..")
+sys.path.append("../legacy_test")
 import auto_parallel_gpt_model as modeling
 from auto_parallel_gpt_model import (
     GPTForPretraining,

python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py → test/distributed_passes/ps_pass_test_base.py

@@ -17,7 +17,7 @@ import shlex
 import sys
 import unittest
 
-from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import (  # noqa: F401
+from dist_pass_test_base import (  # noqa: F401
     prepare_python_path_and_return_module,
     remove_path_if_exists,
 )

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py → test/distributed_passes/test_auto_parallel_amp_pass.py

@@ -13,9 +13,12 @@
 # limitations under the License.
 
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py → test/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py

@@ -17,6 +17,8 @@ import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py → test/distributed_passes/test_auto_parallel_fp16_pass.py

@@ -13,9 +13,12 @@
 # limitations under the License.
 
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py → test/distributed_passes/test_auto_parallel_gradient_merge_pass.py

@@ -14,9 +14,12 @@
 
 import logging
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py → test/distributed_passes/test_auto_parallel_recompute_pass.py

@@ -13,9 +13,12 @@
 # limitations under the License.
 
 import random
+import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py → test/distributed_passes/test_auto_parallel_sharding_pass.py

@@ -17,6 +17,8 @@ import sys
 import unittest
 
 import numpy as np
+
+sys.path.append("../legacy_test")
 from auto_parallel_pass_test_base import AutoPallelPassTestBase
 
 import paddle

python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py → test/distribution/test_dirichlet_op.py

@@ -19,7 +19,7 @@ import scipy.stats
 
 import paddle
 
-sys.path.append("../")
+sys.path.append("../../python/paddle/fluid/tests/unittests")
 import unittest
 
 from eager_op_test import (

python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py → test/ps/ps_dnn_trainer.py

@@ -21,6 +21,7 @@ import sys
 
 import numpy as np
 import yaml
+from ps_dnn_model import StaticModel
 
 import paddle
 from paddle.distributed import fleet
@@ -32,9 +33,6 @@ from paddle.distributed.ps.utils.ps_program_builder import (
     ps_log_root_dir,
 )
 
-sys.path.append("..")
-from ps_dnn_model import StaticModel
-
 __dir__ = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
 

python/paddle/fluid/tests/unittests/ps/test_fl_ps.py → test/ps/test_fl_ps.py

@@ -16,11 +16,11 @@
 
 import os
 import shlex  # noqa: F401
+import sys
 import unittest
 
-from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import (  # noqa: F401
-    remove_path_if_exists,
-)
+sys.path.append("../distributed_passes")
+from dist_pass_test_base import remove_path_if_exists  # noqa: F401
 
 
 class FlPsTest(unittest.TestCase):

python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py → test/ps/test_the_one_ps.py

@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import unittest
 
 from google.protobuf import text_format  # noqa: F401
 
+sys.path.append("../distributed_passes")
+from ps_pass_test_base import PsPassTestBase, remove_path_if_exists
+
 import paddle.distributed.fleet.proto.the_one_ps_pb2 as ps_pb2  # noqa: F401
 from paddle.distributed.ps.utils.public import logger, ps_log_root_dir
-from paddle.fluid.tests.unittests.distributed_passes.ps_pass_test_base import (
-    PsPassTestBase,
-    remove_path_if_exists,
-)
 
 
 class TestTheOnePs(PsPassTestBase):