未验证 提交 43b92b63 编写于 作者: W wangxiaoning 提交者: GitHub

[Fluid clean] (#48105)

* add index sample fp16 support

* remove fluid APIs in distributed_strategy.py and role_maker.py

* Revert "remove fluid APIs in distributed_strategy.py and role_maker.py"

This reverts commit 223bbee990d3bf69e252fc3c0f19e3873550a264.

* remove fluid APIs in distributed_strategy.py and role_maker.py

* remove index sample op changes

* remove fluid APIs under fleet.base

* remove fluid APIs under fleet.layers.mpu

* remove fluid APIs under fleet.meta_optimizers

* fix fluid error

* fix util_factory.py

* reset fluid.io.load_inference_model API
上级 f3a93e51
文件模式从 100644 更改为 100755
文件模式从 100644 更改为 100755
...@@ -103,7 +103,7 @@ class DistributedJobInfo: ...@@ -103,7 +103,7 @@ class DistributedJobInfo:
self.job_info.strategy = dist_strategy self.job_info.strategy = dist_strategy
ReduceStrategyFluid = paddle.fluid.BuildStrategy.ReduceStrategy ReduceStrategyFluid = paddle.static.BuildStrategy.ReduceStrategy
ReduceStrategyFleet = int ReduceStrategyFleet = int
...@@ -207,7 +207,7 @@ class DistributedStrategy: ...@@ -207,7 +207,7 @@ class DistributedStrategy:
strategy.execution_strategy = exe_strategy strategy.execution_strategy = exe_strategy
""" """
execution_strategy = paddle.fluid.ExecutionStrategy() execution_strategy = paddle.static.ExecutionStrategy()
fields = self.strategy.execution_strategy.DESCRIPTOR.fields fields = self.strategy.execution_strategy.DESCRIPTOR.fields
for f in fields: for f in fields:
setattr( setattr(
...@@ -255,7 +255,7 @@ class DistributedStrategy: ...@@ -255,7 +255,7 @@ class DistributedStrategy:
""" """
build_strategy = paddle.fluid.BuildStrategy() build_strategy = paddle.static.BuildStrategy()
fields = self.strategy.build_strategy.DESCRIPTOR.fields fields = self.strategy.build_strategy.DESCRIPTOR.fields
for f in fields: for f in fields:
value = getattr(self.strategy.build_strategy, f.name) value = getattr(self.strategy.build_strategy, f.name)
......
...@@ -19,7 +19,7 @@ import warnings ...@@ -19,7 +19,7 @@ import warnings
from multiprocessing import Process, Manager from multiprocessing import Process, Manager
import paddle import paddle
import paddle.fluid as fluid import paddle.fluid.core as core
from paddle.distributed.fleet.base.private_helper_function import ( from paddle.distributed.fleet.base.private_helper_function import (
wait_server_ready, wait_server_ready,
) )
...@@ -128,7 +128,7 @@ class Gloo: ...@@ -128,7 +128,7 @@ class Gloo:
def _init_fs(self, fs_path, prefix): def _init_fs(self, fs_path, prefix):
def init(rank, nodes, role): def init(rank, nodes, role):
gloo = fluid.core.Gloo() gloo = core.Gloo()
gloo.set_rank(rank) gloo.set_rank(rank)
gloo.set_size(nodes) gloo.set_size(nodes)
gloo.set_prefix(prefix) gloo.set_prefix(prefix)
...@@ -156,7 +156,7 @@ class Gloo: ...@@ -156,7 +156,7 @@ class Gloo:
def _init_dfs(self, dfs_name, dfs_ugi, dfs_path, prefix): def _init_dfs(self, dfs_name, dfs_ugi, dfs_path, prefix):
def init(rank, nodes, role): def init(rank, nodes, role):
gloo = fluid.core.Gloo() gloo = core.Gloo()
gloo.set_rank(rank) gloo.set_rank(rank)
gloo.set_size(nodes) gloo.set_size(nodes)
gloo.set_prefix(prefix) gloo.set_prefix(prefix)
...@@ -216,7 +216,7 @@ class Gloo: ...@@ -216,7 +216,7 @@ class Gloo:
return _http_server return _http_server
def init(rank, nodes, role): def init(rank, nodes, role):
gloo = fluid.core.Gloo() gloo = core.Gloo()
gloo.set_rank(rank) gloo.set_rank(rank)
gloo.set_size(nodes) gloo.set_size(nodes)
gloo.set_prefix(prefix) gloo.set_prefix(prefix)
...@@ -1175,7 +1175,7 @@ class PaddleCloudRoleMaker(RoleMakerBase): ...@@ -1175,7 +1175,7 @@ class PaddleCloudRoleMaker(RoleMakerBase):
else: else:
self._collective_env() self._collective_env()
self._role_is_generated = True self._role_is_generated = True
if not paddle.fluid.framework._non_static_mode(): if not paddle.framework.in_dynamic_mode():
self._gloo_init() self._gloo_init()
......
...@@ -16,12 +16,13 @@ ...@@ -16,12 +16,13 @@
"""basic collective operations in python""" """basic collective operations in python"""
"""remote file system""" """remote file system"""
import paddle
from ..utils.fs import FS from ..utils.fs import FS
from paddle.fluid.proto import framework_pb2 from paddle.fluid.proto import framework_pb2
from paddle.fluid.framework import Program from paddle.static import Program
from paddle.fluid import debugger from paddle.fluid import debugger
from google.protobuf import text_format from google.protobuf import text_format
import paddle.fluid as fluid import paddle.framework as framework
from collections import OrderedDict from collections import OrderedDict
from paddle.fluid import core from paddle.fluid import core
import subprocess import subprocess
...@@ -376,7 +377,7 @@ class UtilBase: ...@@ -376,7 +377,7 @@ class UtilBase:
pruned_vars = [ pruned_vars = [
(v.name, v) (v.name, v)
for v in pruned_prog.list_vars() for v in pruned_prog.list_vars()
if fluid.io.is_persistable(v) if paddle.static.io.is_persistable(v)
] ]
pruned_vars = OrderedDict(pruned_vars) pruned_vars = OrderedDict(pruned_vars)
pruned_vars_name = [name for name in pruned_vars] pruned_vars_name = [name for name in pruned_vars]
...@@ -460,7 +461,7 @@ class UtilBase: ...@@ -460,7 +461,7 @@ class UtilBase:
) )
saved_params = [ saved_params = [
v for v in prog.list_vars() if fluid.io.is_persistable(v) v for v in prog.list_vars() if paddle.static.io.is_persistable(v)
] ]
print( print(
"persistable vars in dump program: {}".format( "persistable vars in dump program: {}".format(
...@@ -487,15 +488,15 @@ class UtilBase: ...@@ -487,15 +488,15 @@ class UtilBase:
) )
return False return False
place = fluid.CPUPlace() place = framework.CPUPlace()
exe = fluid.Executor(place) exe = paddle.static.Executor(place)
scope = fluid.core.Scope() scope = paddle.static.Scope()
with fluid.scope_guard(scope): with paddle.static.scope_guard(scope):
( (
inference_program, inference_program,
feed_target_names, feed_target_names,
fetch_targets, fetch_targets,
) = fluid.io.load_inference_model( ) = paddle.fluid.io.load_inference_model(
config.dump_model_dir, config.dump_model_dir,
exe, exe,
model_filename=model_filename, model_filename=model_filename,
...@@ -508,7 +509,7 @@ class UtilBase: ...@@ -508,7 +509,7 @@ class UtilBase:
for each_var in saved_params for each_var in saved_params
} }
for each_var in saved_params: for each_var in saved_params:
var_temp = fluid.global_scope().find_var(each_var.name) var_temp = paddle.static.global_scope().find_var(each_var.name)
assert var_temp is not None, ( assert var_temp is not None, (
"can't not find var: " + each_var.name "can't not find var: " + each_var.name
) )
...@@ -639,7 +640,7 @@ class UtilBase: ...@@ -639,7 +640,7 @@ class UtilBase:
dtype=feed_config.feeded_vars_types[i], dtype=feed_config.feeded_vars_types[i],
) )
feed_tensors.append( feed_tensors.append(
fluid.create_lod_tensor( paddle.fluid.create_lod_tensor(
t, [[1] * config.batch_size], place t, [[1] * config.batch_size], place
) )
) )
...@@ -668,7 +669,9 @@ class UtilBase: ...@@ -668,7 +669,9 @@ class UtilBase:
) )
for i in range(len(feed_config.feeded_vars_names)) for i in range(len(feed_config.feeded_vars_names))
] ]
feeder = fluid.DataFeeder(feed_list=feed_vars, place=place) feeder = paddle.fluid.DataFeeder(
feed_list=feed_vars, place=place
)
batch_feed = feed_gen( batch_feed = feed_gen(
config.batch_size, config.batch_size,
feed_config.feeded_vars_dims, feed_config.feeded_vars_dims,
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
import paddle import paddle
from . import mp_ops from . import mp_ops
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid.dygraph.layers import Layer from paddle.nn import Layer
from .random import get_rng_state_tracker from .random import get_rng_state_tracker
from paddle.nn import functional as F from paddle.nn import functional as F
from ...base import topology as tp from ...base import topology as tp
......
...@@ -15,17 +15,17 @@ ...@@ -15,17 +15,17 @@
import paddle import paddle
from paddle import _legacy_C_ops from paddle import _legacy_C_ops
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid.framework import _non_static_mode from paddle.framework import in_dynamic_mode
from paddle.fluid.framework import _in_legacy_dygraph from paddle.framework import _in_legacy_dygraph
from paddle.fluid.framework import in_dygraph_mode from paddle.framework import in_dygraph_mode
from paddle.fluid.framework import _varbase_creator from paddle.framework import _varbase_creator
from paddle.fluid.layer_helper import LayerHelper from paddle.framework import LayerHelper
from paddle.fluid.data_feeder import check_variable_and_dtype from paddle.fluid.data_feeder import check_variable_and_dtype
from paddle.fluid.dygraph import layers from paddle.nn import Layer
from paddle.distributed import collective from paddle.distributed import collective
from ....communication.reduce import ReduceOp, _get_reduce_op from ....communication.reduce import ReduceOp, _get_reduce_op
from paddle.fluid.data_feeder import check_dtype from paddle.fluid.data_feeder import check_dtype
import paddle.fluid.dygraph_utils as dygraph_utils from paddle.common_ops_import import dygraph_utils
def _c_identity(tensor, group=None): def _c_identity(tensor, group=None):
...@@ -123,7 +123,7 @@ def _c_concat(tensor, group=None): ...@@ -123,7 +123,7 @@ def _c_concat(tensor, group=None):
rank = group.rank rank = group.rank
nranks = group.nranks nranks = group.nranks
if _non_static_mode(): if in_dynamic_mode():
return _legacy_C_ops.c_concat( return _legacy_C_ops.c_concat(
tensor, tensor,
'ring_id', 'ring_id',
...@@ -189,7 +189,7 @@ def _c_split(tensor, group=None): ...@@ -189,7 +189,7 @@ def _c_split(tensor, group=None):
else group.nranks else group.nranks
) )
if _non_static_mode(): if in_dynamic_mode():
return _legacy_C_ops.c_split( return _legacy_C_ops.c_split(
tensor, tensor,
'use_calc_stream', 'use_calc_stream',
...@@ -335,7 +335,7 @@ def _c_lookup_table(table, index, start_index=0, name=None): ...@@ -335,7 +335,7 @@ def _c_lookup_table(table, index, start_index=0, name=None):
Returns: Returns:
Tensor. Tensor.
""" """
if _non_static_mode(): if in_dynamic_mode():
return _legacy_C_ops.c_embedding( return _legacy_C_ops.c_embedding(
table, index, "start_index", start_index table, index, "start_index", start_index
) )
...@@ -354,7 +354,7 @@ def _c_lookup_table(table, index, start_index=0, name=None): ...@@ -354,7 +354,7 @@ def _c_lookup_table(table, index, start_index=0, name=None):
return tmp return tmp
class _Linear(layers.Layer): class _Linear(Layer):
""" """
Linear Linear
""" """
...@@ -424,7 +424,7 @@ def _c_softmax_with_cross_entropy( ...@@ -424,7 +424,7 @@ def _c_softmax_with_cross_entropy(
if input_dims - 1 == label_dims: if input_dims - 1 == label_dims:
label = paddle.unsqueeze(label, axis=-1) label = paddle.unsqueeze(label, axis=-1)
if _non_static_mode(): if in_dynamic_mode():
softmax, loss = _legacy_C_ops.c_softmax_with_cross_entropy( softmax, loss = _legacy_C_ops.c_softmax_with_cross_entropy(
logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks
) )
...@@ -458,7 +458,7 @@ def _linear(x, weight, bias=None, name=None): ...@@ -458,7 +458,7 @@ def _linear(x, weight, bias=None, name=None):
""" """
Fuction Linear Fuction Linear
""" """
if _non_static_mode(): if in_dynamic_mode():
pre_bias = _varbase_creator(dtype=x.dtype) pre_bias = _varbase_creator(dtype=x.dtype)
_legacy_C_ops.matmul( _legacy_C_ops.matmul(
x, x,
...@@ -825,7 +825,7 @@ def split( ...@@ -825,7 +825,7 @@ def split(
supported_operations supported_operations
) )
) )
if _non_static_mode(): if in_dynamic_mode():
raise ValueError( raise ValueError(
"paddle.distributed.split cannot be used in dynamic " "paddle.distributed.split cannot be used in dynamic "
"graph mode, plese use ParallelEmbedding, ParallelRowLinear, " "graph mode, plese use ParallelEmbedding, ParallelRowLinear, "
......
...@@ -18,8 +18,9 @@ import contextlib ...@@ -18,8 +18,9 @@ import contextlib
from paddle import _legacy_C_ops from paddle import _legacy_C_ops
from paddle.fluid import core from paddle.fluid import core
from paddle.fluid.data_feeder import check_variable_and_dtype from paddle.fluid.data_feeder import check_variable_and_dtype
from paddle.fluid.framework import Variable, _non_static_mode from paddle.static import Variable
from paddle.fluid.layer_helper import LayerHelper from paddle.framework import in_dynamic_mode
from paddle.framework import LayerHelper
__all__ = [] __all__ = []
...@@ -209,7 +210,7 @@ def dropout( ...@@ -209,7 +210,7 @@ def dropout(
) # semantic transfer ) # semantic transfer
# dygraph using tracker, doesn't need determinate seed # dygraph using tracker, doesn't need determinate seed
if _non_static_mode(): if in_dynamic_mode():
out, mask = _legacy_C_ops.dropout( out, mask = _legacy_C_ops.dropout(
x, x,
'dropout_prob', 'dropout_prob',
......
...@@ -11,8 +11,9 @@ ...@@ -11,8 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
import paddle.fluid as fluid import paddle.static as static
from paddle.fluid import core, unique_name from paddle.fluid import core
from paddle.utils import unique_name
from .meta_optimizer_base import MetaOptimizerBase from .meta_optimizer_base import MetaOptimizerBase
from .common import ( from .common import (
OpRole, OpRole,
...@@ -132,7 +133,7 @@ class RawProgramOptimizer(MetaOptimizerBase): ...@@ -132,7 +133,7 @@ class RawProgramOptimizer(MetaOptimizerBase):
self.rank = self.role_maker._worker_index() self.rank = self.role_maker._worker_index()
self.nranks = self.role_maker._worker_num() self.nranks = self.role_maker._worker_num()
if startup_program is None: if startup_program is None:
startup_program = fluid.default_startup_program() startup_program = static.default_startup_program()
self.startup_program = startup_program self.startup_program = startup_program
block = loss.block block = loss.block
......
...@@ -13,10 +13,11 @@ ...@@ -13,10 +13,11 @@
# limitations under the License. # limitations under the License.
import os import os
from paddle.fluid import unique_name, core from paddle.fluid import core
import paddle.fluid as fluid from paddle.utils import unique_name
from paddle.fluid.optimizer import PipelineOptimizer
from paddle.static import default_startup_program, device_guard from paddle.static import default_startup_program, device_guard
from paddle.fluid import layers from paddle.static import create_global_var
from .common import OpRole, OP_ROLE_VAR_KEY, CollectiveHelper, OP_ROLE_KEY from .common import OpRole, OP_ROLE_VAR_KEY, CollectiveHelper, OP_ROLE_KEY
from .common import is_backward_op, is_optimizer_op, is_update_op from .common import is_backward_op, is_optimizer_op, is_update_op
...@@ -275,7 +276,7 @@ class ShardingOptimizer(MetaOptimizerBase): ...@@ -275,7 +276,7 @@ class ShardingOptimizer(MetaOptimizerBase):
) )
if self.pp_degree > 1: if self.pp_degree > 1:
pp_optimizer = fluid.optimizer.PipelineOptimizer( pp_optimizer = PipelineOptimizer(
self.inner_opt, self._gradient_merge_acc_step self.inner_opt, self._gradient_merge_acc_step
) )
self._pp_optimizer = pp_optimizer self._pp_optimizer = pp_optimizer
...@@ -1916,7 +1917,7 @@ class ShardingOptimizer(MetaOptimizerBase): ...@@ -1916,7 +1917,7 @@ class ShardingOptimizer(MetaOptimizerBase):
def _create_gm_cond(self, main_block): def _create_gm_cond(self, main_block):
# Add const var # Add const var
acc_step_var = layers.create_global_var( acc_step_var = create_global_var(
name="gradient_merge_acc_step", name="gradient_merge_acc_step",
shape=[1], shape=[1],
value=int(self._gradient_merge_acc_step), value=int(self._gradient_merge_acc_step),
...@@ -1925,7 +1926,7 @@ class ShardingOptimizer(MetaOptimizerBase): ...@@ -1925,7 +1926,7 @@ class ShardingOptimizer(MetaOptimizerBase):
force_cpu=True, force_cpu=True,
) )
zero_var = layers.create_global_var( zero_var = create_global_var(
name="gradient_merge_zero", name="gradient_merge_zero",
shape=[1], shape=[1],
value=int(0), value=int(0),
...@@ -1935,7 +1936,7 @@ class ShardingOptimizer(MetaOptimizerBase): ...@@ -1935,7 +1936,7 @@ class ShardingOptimizer(MetaOptimizerBase):
) )
# Add step var & cond var # Add step var & cond var
current_step_var = layers.create_global_var( current_step_var = create_global_var(
name="gradient_merge_current_step", name="gradient_merge_current_step",
shape=[1], shape=[1],
value=int(0), value=int(0),
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
import paddle.fluid as fluid import paddle.static as static
from .meta_optimizer_base import MetaOptimizerBase from .meta_optimizer_base import MetaOptimizerBase
from .common import ( from .common import (
CollectiveHelper, CollectiveHelper,
...@@ -174,7 +174,7 @@ class TensorParallelOptimizer(MetaOptimizerBase): ...@@ -174,7 +174,7 @@ class TensorParallelOptimizer(MetaOptimizerBase):
self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.current_endpoint = self.endpoints[self.role_maker._worker_index()]
self.startup_program = startup_program self.startup_program = startup_program
if startup_program is None: if startup_program is None:
self.startup_program = fluid.default_startup_program() self.startup_program = static.default_startup_program()
optimize_ops, params_grads = self.inner_opt.minimize( optimize_ops, params_grads = self.inner_opt.minimize(
loss, self.startup_program, parameter_list, no_grad_set loss, self.startup_program, parameter_list, no_grad_set
......
文件模式从 100644 更改为 100755
文件模式从 100644 更改为 100755
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册