[RM FLUID] rm fluid_pslib_init (#50549)

* [RM FLUID] rm fluid_pslib_init

* [RM FLUID] for ci

* [RM FLUID] for ci

paddle/fluid/distributed/CMakeLists.txt

@@ -7,7 +7,7 @@ if(WITH_PYTHON)
   file(MAKE_DIRECTORY
        ${PADDLE_BINARY_DIR}/python/paddle/distributed/fleet/proto)
   set(PSLIB_PROTO_DSTPATH
-      "${PADDLE_SOURCE_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/"
+      "${PADDLE_SOURCE_DIR}/python/paddle/incubate/fleet/parameter_server/pslib/"
   )
   if(NOT WIN32)
     add_custom_command(

python/paddle/fluid/dataset.py

@@ -867,7 +867,7 @@ class InMemoryDataset(DatasetBase):
 
               # required: skiptest
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
               filelist = ["a.txt", "b.txt"]
               dataset.set_filelist(filelist)
@@ -929,7 +929,7 @@ class InMemoryDataset(DatasetBase):
 
               # required: skiptest
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
               filelist = ["a.txt", "b.txt"]
               dataset.set_filelist(filelist)
@@ -993,7 +993,7 @@ class InMemoryDataset(DatasetBase):
 
               # required: skiptest
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
               filelist = ["a.txt", "b.txt"]
               dataset.set_filelist(filelist)
@@ -1037,7 +1037,7 @@ class InMemoryDataset(DatasetBase):
 
               # required: skiptest
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
               filelist = ["a.txt", "b.txt"]
               dataset.set_filelist(filelist)
@@ -1084,7 +1084,7 @@ class InMemoryDataset(DatasetBase):
 
               # required: skiptest
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
               graph_config = {"walk_len": 24,
                     "walk_degree": 10,
@@ -1251,9 +1251,9 @@ class QueueDataset(DatasetBase):
             .. code-block:: python
 
               import paddle.fluid as fluid
-              from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+              from paddle.incubate.fleet.parameter_server.pslib import fleet
               dataset = fluid.DatasetFactory().create_dataset("QueueDataset")
-              dataset.global_shuffle(fleet)
+              #dataset.global_shuffle(fleet)
 
         Raises:
             NotImplementedError: QueueDataset does not support global shuffle

python/paddle/fluid/executor.py

@@ -2126,7 +2126,7 @@ class Executor:
         assert len(fetch_list) == len(fetch_info)
         compiled = isinstance(program, compiler.CompiledProgram)
         if is_heter:
-            from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+            from paddle.incubate.fleet.parameter_server.pslib import fleet
             from paddle.fluid.incubate.fleet.utils.fleet_util import FleetUtil
 
             fu = FleetUtil()

python/paddle/fluid/incubate/fleet/utils/fleet_util.py

@@ -57,7 +57,7 @@ class FleetUtil:
         op_maker = core.op_proto_and_checker_maker
         self.op_role_key = op_maker.kOpRoleAttrName()
         if mode == "pslib":
-            from paddle.fluid.incubate.fleet.parameter_server.pslib import (
+            from paddle.incubate.fleet.parameter_server.pslib import (
                 fleet as fleet_pslib,
             )
 

python/paddle/fluid/tests/unittests/test_dataset.py

@@ -1185,7 +1185,7 @@ class TestDataset2(unittest.TestCase):
         train_program = fluid.Program()
         startup_program = fluid.Program()
         scope = fluid.Scope()
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         with fluid.program_guard(train_program, startup_program):
             slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"]
@@ -1316,7 +1316,7 @@ class TestDataset2(unittest.TestCase):
         train_program = fluid.Program()
         startup_program = fluid.Program()
         scope = fluid.Scope()
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         with fluid.program_guard(train_program, startup_program):
             slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"]

python/paddle/fluid/tests/unittests/test_downpoursgd.py

@@ -21,12 +21,12 @@ from google.protobuf import text_format
 
 import paddle
 import paddle.fluid as fluid
-import paddle.fluid.incubate.fleet.parameter_server.pslib.ps_pb2 as pslib
-from paddle.fluid.incubate.fleet.parameter_server.pslib.node import (
+import paddle.incubate.fleet.parameter_server.pslib.ps_pb2 as pslib
+from paddle.fluid.trainer_factory import TrainerFactory
+from paddle.incubate.fleet.parameter_server.pslib.node import (
     DownpourServer,
     DownpourWorker,
 )
-from paddle.fluid.trainer_factory import TrainerFactory
 
 cache_path = os.path.expanduser('~/.cache/paddle/dataset')
 

python/paddle/fluid/tests/unittests/test_fleet.py

@@ -35,7 +35,7 @@ class TestFleet1(unittest.TestCase):
         import paddle
         import paddle.fluid as fluid
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"

python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py

@@ -35,7 +35,7 @@ class TestFleet1(unittest.TestCase):
         """Test cases for pslib."""
         import paddle.fluid as fluid
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py

@@ -64,7 +64,7 @@ class TestCloudRoleMaker(unittest.TestCase):
         """Test cases for pslib."""
         import paddle.fluid as fluid
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"

python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py

@@ -35,7 +35,7 @@ class TestCloudRoleMaker(unittest.TestCase):
         """Test cases for pslib."""
         import paddle.fluid as fluid
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"

python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py

@@ -35,7 +35,7 @@ class TestFleet1(unittest.TestCase):
         """Test cases for pslib."""
         import paddle.fluid as fluid
         from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
-        from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
+        from paddle.incubate.fleet.parameter_server.pslib import fleet
 
         os.environ["POD_IP"] = "127.0.0.1"
         os.environ["PADDLE_PORT"] = "36001"

python/paddle/incubate/fleet/parameter_server/pslib/__init__.py

@@ -1230,7 +1230,7 @@ class DownpourOptimizer(DistributedOptimizer):
         fleet._main_programs = programs
         fleet._scopes = scopes
         if opt_info["use_ps_gpu"]:
-            from paddle.fluid.transpiler.collective import MultiThread
+            from paddle.distributed.transpiler.collective import MultiThread
 
             # check start program
             if program_mode not in [

python/setup.py.in

@@ -406,7 +406,6 @@ packages=['paddle',
           'paddle.fluid.incubate.fleet.base',
           'paddle.fluid.incubate.fleet.parameter_server',
           'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
-          'paddle.fluid.incubate.fleet.parameter_server.pslib',
           'paddle.fluid.incubate.fleet.parameter_server.ir',
           'paddle.fluid.incubate.fleet.collective',
           'paddle.fluid.incubate.fleet.utils',
@@ -438,6 +437,7 @@ packages=['paddle',
           'paddle.incubate.distributed.models.moe',
           'paddle.incubate.distributed.models.moe.gate',
           'paddle.incubate.fleet.parameter_server.distribute_transpiler',
+          'paddle.incubate.fleet.parameter_server.pslib',
           'paddle.quantization',
           'paddle.quantization.quanters',
           'paddle.quantization.observers',
@@ -594,7 +594,7 @@ if '${WITH_CINN}' == 'ON':
 if '${WITH_PSLIB}' == 'ON':
     shutil.copy('${PSLIB_LIB}', libs_path)
     if os.path.exists('${PSLIB_VERSION_PY}'):
-        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/')
+        shutil.copy('${PSLIB_VERSION_PY}', '${PADDLE_BINARY_DIR}/python/paddle/incubate/fleet/parameter_server/pslib/')
     package_data['paddle.libs'] += ['libps' + ext_name]
 
 if '${WITH_MKLDNN}' == 'ON':

setup.py

@@ -963,7 +963,7 @@ def get_package_data_and_package_dir():
             shutil.copy(
                 env_dict.get("PSLIB_VERSION_PY"),
                 paddle_binary_dir
-                + '/python/paddle/fluid/incubate/fleet/parameter_server/pslib/',
+                + '/python/paddle/incubate/fleet/parameter_server/pslib/',
             )
         package_data['paddle.libs'] += ['libps' + ext_suffix]
     if env_dict.get("WITH_MKLDNN") == 'ON':
@@ -1293,7 +1293,6 @@ def get_setup_parameters():
         'paddle.fluid.incubate.fleet.base',
         'paddle.fluid.incubate.fleet.parameter_server',
         'paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler',
-        'paddle.fluid.incubate.fleet.parameter_server.pslib',
         'paddle.fluid.incubate.fleet.parameter_server.ir',
         'paddle.fluid.incubate.fleet.collective',
         'paddle.fluid.incubate.fleet.utils',
@@ -1325,6 +1324,7 @@ def get_setup_parameters():
         'paddle.incubate.distributed.models.moe',
         'paddle.incubate.distributed.models.moe.gate',
         'paddle.incubate.fleet.parameter_server.distribute_transpiler',
+        'paddle.incubate.fleet.parameter_server.pslib',
         'paddle.quantization',
         'paddle.quantization.quanters',
         'paddle.quantization.observers',