framework.py 259.2 KB
Newer Older
G
gouzil 已提交
1
#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Y
Yu Yang 已提交
15
import collections
X
Xin Pan 已提交
16
from collections import defaultdict
H
huzhiqiang 已提交
17
from collections.abc import Iterable
Q
qiaolongfei 已提交
18
import contextlib
19
from .wrapped_decorator import signature_safe_contextmanager, wrap_decorator
P
peizhilin 已提交
20
import os
F
fengjiayi 已提交
21
import re
22
import traceback
23
import six
24
import copy
25
from types import MethodType, FunctionType
26

Y
Yu Yang 已提交
27
import numpy as np
28
import subprocess
S
sneaxiy 已提交
29
import multiprocessing
30
import sys
31
import logging
M
minqiyang 已提交
32
from .. import compat as cpt
33
from .proto import framework_pb2
34 35

from . import core
36
from . import unique_name
37 38
import paddle.version as fluid_version
import warnings
39
import functools
40
from .variable_index import _getitem_impl_, _setitem_impl_
Y
Yu Yang 已提交
41

42
__all__ = [
43 44 45 46
    'Program',
    'default_startup_program',
    'default_main_program',
    'program_guard',
47
    'name_scope',
J
jianghaicheng 已提交
48
    'ipu_shard_guard',
49
    'set_ipu_shard',
S
sneaxiy 已提交
50 51
    'cuda_places',
    'cpu_places',
52
    'xpu_places',
53
    'mlu_places',
S
sneaxiy 已提交
54
    'cuda_pinned_places',
J
Jiabin Yang 已提交
55
    '_non_static_mode',
L
lujun 已提交
56
    'in_dygraph_mode',
57
    'is_compiled_with_cinn',
C
chengduo 已提交
58
    'is_compiled_with_cuda',
59
    'is_compiled_with_rocm',
60
    'is_compiled_with_xpu',
61
    'is_compiled_with_npu',
62
    'Variable',
63
    'require_version',
64
    'device_guard',
G
guofei 已提交
65 66
    'set_flags',
    'get_flags',
67
]
Y
Yu Yang 已提交
68

Q
qiaolongfei 已提交
69 70 71 72
EMPTY_VAR_NAME = core.kEmptyVarName()
TEMP_VAR_NAME = core.kTempVarName()
GRAD_VAR_SUFFIX = core.kGradVarSuffix()
ZERO_VAR_SUFFIX = core.kZeroVarSuffix()
W
Wu Yi 已提交
73 74
CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName()

L
lujun 已提交
75
_dygraph_tracer_ = None
76
_in_eager_mode_ = (os.environ.get('FLAGS_enable_eager_mode', '1') == '1')
77
_global_expected_place_ = None
78
_current_device = None
79
global_prog_seed = 0
80
_current_pipeline_stage = None
81
_already_patch_eager_tensor = False
J
Jiabin Yang 已提交
82
_already_patch_varbase = False
83
_current_cuda_graph_mode = None
84
_global_flags_ = core.globals()
L
Leo Chen 已提交
85 86
_enable_standalone_executor_ = (os.environ.get('FLAGS_USE_STANDALONE_EXECUTOR',
                                               None))
87
_dy2st_enable_standalone_executor_ = (os.environ.get(
88
    'FLAGS_DY2ST_USE_STANDALONE_EXECUTOR', 1))
J
Jiabin Yang 已提交
89 90

# Some explanation of our execution system 2022.03
91
# For now we have 3 kinds of execution system, since we refactored dygraph mode to
J
Jiabin Yang 已提交
92
# build a fast execution system for dynamic mode. But we can't just remove all legacy
93
# code once we present the new system for some historical reason. That's why we have
J
Jiabin Yang 已提交
94
# these flags.
95
#
J
Jiabin Yang 已提交
96
# 1. _non_static_mode():
97
# _non_static_mode means  we are now running in legacy dygraph mode or dygraph mode.
J
Jiabin Yang 已提交
98 99 100 101
# 2. dygraph_mode():
# This flags inidicates we are now running in dygraph mode which called eager mode before.
# 3. _in_legacy_dygraph():
# This flags inidicates we are now running in legacy dygraph mode
102
#
J
Jiabin Yang 已提交
103
# They have a relation ship as below:
104
# Both dygraph_mode and _in_legacy_dygraph are _non_static_mode, but if you are running in
J
Jiabin Yang 已提交
105
# dygraph mode means you are not in _in_legacy_dygraph.
106
#
J
Jiabin Yang 已提交
107 108 109 110 111 112
# Why we have to make different of _in_legacy_dygraph and dygraph_mode?
# In some performance issue, we find that python if statement cause server performance problem
# and we need our new dygraph mode becomes as fast as it could be. That's why we make these flags
# to make sure in most case, we find new dygraph mode first with only one if statement.


113 114 115 116 117
def _update_monkey_methods(is_eager):
    """
    Update monkey methods of VarBase or eager.Tensor while
    switching eager mode and legacy mode.
    """
118
    from paddle import _C_ops, _legacy_C_ops
119 120 121
    from .dygraph.varbase_patch_methods import monkey_patch_varbase
    from .dygraph import monkey_patch_math_varbase

122 123 124
    global _already_patch_eager_tensor
    global _already_patch_varbase

125
    assert isinstance(is_eager, bool)
126
    # switch into eager mode
127
    if is_eager:
128
        _legacy_C_ops.switch_to_eager_ops()
129 130 131 132 133 134
        if not _already_patch_eager_tensor:
            monkey_patch_varbase()
            monkey_patch_math_varbase()

            _already_patch_eager_tensor = True
    # switch back into legacy mode
135
    else:
136
        _legacy_C_ops.switch_to_core_ops()
137 138 139 140 141
        if not _already_patch_varbase:
            monkey_patch_varbase()
            monkey_patch_math_varbase()

            _already_patch_varbase = True
142

143 144 145 146 147 148 149 150 151 152 153
    # switch Paddle.Tensor bind type
    _switch_tensor_bind_type(is_eager)


def _switch_tensor_bind_type(is_eager):
    import paddle
    if is_eager:
        paddle.Tensor = core.eager.Tensor
    else:
        paddle.Tensor = core.VarBase
    paddle.Tensor.__qualname__ = 'Tensor'
154 155


J
Jiabin Yang 已提交
156 157 158
def _enable_legacy_dygraph():
    global _in_eager_mode_
    _in_eager_mode_ = False
159
    _update_monkey_methods(is_eager=False)
J
Jiabin Yang 已提交
160 161 162 163 164


def _disable_legacy_dygraph():
    global _in_eager_mode_
    _in_eager_mode_ = True
165
    _update_monkey_methods(is_eager=True)
J
Jiabin Yang 已提交
166 167 168 169 170 171 172


def _in_eager_without_dygraph_check():
    global _in_eager_mode_
    return _in_eager_mode_


173 174 175 176 177 178 179 180 181
# FIXME(dev): We haven't fully verified eager mode on XPU/NPU et.al but
# only GPU/CPU. Remove this after we improve this feature.
_is_first_import_ = True


def _fallback_legacy_dygraph():
    global _in_eager_mode_
    global _is_first_import_
    need_fallback = False
C
Chen Weihang 已提交
182 183 184
    # Only enable eager on CPU/GPU/XPU
    is_not_support = core.is_compiled_with_npu() or core.is_compiled_with_ipu(
    ) or core.is_compiled_with_mlu()
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205

    if _in_eager_mode_ and is_not_support:
        # switch into legacy dygraph mode
        warnings.warn(
            "We will fallback into legacy dygraph on NPU/XPU/MLU/IPU/ROCM devices. Because we only support new eager dygraph mode on CPU/GPU currently. "
        )
        _in_eager_mode_ = False
        if not _is_first_import_:
            _enable_legacy_dygraph()
        need_fallback = True

    need_fallback = False
    _is_first_import_ = False

    return need_fallback


# switch into legacy mode if need while import paddle
_fallback_legacy_dygraph()


J
Jiabin Yang 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
def in_dygraph_mode():
    """

    .. note::
        Dynamic graph mode is turn ON by default since paddle 2.0.0

    This API checks whether paddle runs in dynamic graph mode.

    You can turn ON static graph mode by `enable_static <../dygraph/base/disable_dygraph_en.html>`_ ,
    and turn OFF static graph mode by `disable_static <../dygraph/base/enable_dygraph_en.html>`_  .

    Returns:
        bool: Whether paddle runs in dynamic graph mode.

    Examples:
        .. code-block:: python

            import paddle
            print(paddle.in_dynamic_mode())  # True, dynamic mode is turn ON by default since paddle 2.0.0

            paddle.enable_static()
            print(paddle.in_dynamic_mode())  # False, Now we are in static mode

            paddle.disable_static()
            print(paddle.in_dynamic_mode())  # True, Now we are in dynamic mode

    """
    return (_dygraph_tracer_ is not None) and _in_eager_mode_


def _in_legacy_dygraph():
    return (not _in_eager_mode_) and (_dygraph_tracer_ is not None)


def _non_static_mode():
    return _dygraph_tracer_ is not None
242 243 244


@signature_safe_contextmanager
J
Jiabin Yang 已提交
245
def _test_eager_guard(place=None):
C
Chen Weihang 已提交
246 247
    # FIXME(dev): We haven't fully verified eager mode on NPU et.al but
    # only GPU/CPU/XPU. Remove this after we improve this feature.
248 249 250
    already_fallback = _fallback_legacy_dygraph()
    if not already_fallback:
        _disable_legacy_dygraph()
251
    try:
J
Jiabin Yang 已提交
252
        yield
253
    finally:
254 255
        if not already_fallback:
            _enable_legacy_dygraph()
256 257


258 259
global_ipu_index = -1
global_ipu_stage = -1
J
jianghaicheng 已提交
260 261 262 263
ipu_index_attr_name = 'ipu_index'
ipu_stage_attr_name = 'ipu_stage'


L
Leo Chen 已提交
264 265 266 267 268 269 270 271 272 273 274
@signature_safe_contextmanager
def _enable_standalone_executor(enable=True):
    global _enable_standalone_executor_
    original_ = _enable_standalone_executor_
    _enable_standalone_executor_ = enable
    try:
        yield
    finally:
        _enable_standalone_executor_ = original_


J
jianghaicheng 已提交
275
@signature_safe_contextmanager
276
def ipu_shard_guard(index=-1, stage=-1):
J
jianghaicheng 已提交
277 278 279 280
    """
    Used to shard the graph on IPUs. Set each Op run on which IPU in the sharding and which stage in the pipelining.

    Args:
W
Weilong Wu 已提交
281
        index(int, optional): Specify which ipu the Tensor is computed on, (such as '0, 1, 2, 3').
282
            The default value is -1, which means the Op only run on IPU 0.
W
Weilong Wu 已提交
283
        stage(int, optional): Specify the computation order of the sharded model(such as '0, 1, 2, 3').
284
            The sharded model will be computed from small to large. The default value is -1,
J
jianghaicheng 已提交
285
            which means no pipelining computation order and run Ops in terms of graph.
286

G
gouzil 已提交
287 288 289 290 291 292 293
    Note:
        Only if the enable_manual_shard=True, the 'index' is able to be set not -1. Please refer
        to :ref:`api_paddle_static_IpuStrategy`.
        Only if the enable_pipelining=True, the 'stage' is able to be set not -1. Please refer
        to :ref:`api_paddle_static_IpuStrategy`.
        A index is allowed to match none stage or a stage. A stage is only allowed to match a new or
        duplicated index.
J
jianghaicheng 已提交
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327

    Examples:
        .. code-block:: python

            # required: ipu

            import paddle
            paddle.enable_static()
            a = paddle.static.data(name='data', shape=[None, 1], dtype='int32')
            with paddle.static.ipu_shard_guard(index=0, stage=0):
                b = a + 1
            with paddle.static.ipu_shard_guard(index=1, stage=1):
                c = b + 1
            with paddle.static.ipu_shard_guard(index=0, stage=2):
                d = c + 1
    """
    if not core.is_compiled_with_ipu():
        raise ValueError(
            "Can not use this function since PaddlePaddle is not compiled with IPU"
        )

    global global_ipu_index
    global global_ipu_stage
    prev_ipu_index = global_ipu_index
    prev_ipu_stage = global_ipu_stage
    global_ipu_index = index
    global_ipu_stage = stage
    try:
        yield
    finally:
        global_ipu_index = prev_ipu_index
        global_ipu_stage = prev_ipu_stage


328 329 330 331
def set_ipu_shard(call_func, index=-1, stage=-1):
    """
    Shard the ipu with the given call function. Set every ops in call function to the given ipu sharding.

G
gouzil 已提交
332 333 334 335 336
    Note:
        Only when enable_manual_shard=True to set the index to a value other than -1. please refer to :ref:`api_paddle_static_IpuStrategy` .
        Only when enable_pipelining=True to set stage to a value other than -1. please refer to :ref:`api_paddle_static_IpuStrategy` .
        An index supports a corresponding None stage or a stage, and a stage only supports a new index or a duplicate index.

337 338 339 340 341
    Args:
        call_func(Layer|function): Specify the call function to be wrapped.
        index(int, optional): Specify which ipu the Tensor is computed on, (such as ‘0, 1, 2, 3’).
            The default value is -1, which means the Op only run on IPU 0.
        stage(int, optional): Specify the computation order of the sharded model(such as ‘0, 1, 2, 3’).
342
            The sharded model will be computed from small to large. The default value is -1,
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
            which means no pipelining computation order and run Ops in terms of graph.

    Returns:
        The wrapped call function.

    Examples:
        .. code-block:: python

            # required: ipu

            import paddle
            paddle.enable_static()
            a = paddle.static.data(name='data', shape=[None, 1], dtype='float32')
            relu = paddle.nn.ReLU()
            relu = paddle.static.set_ipu_shard(relu, index=1, stage=1)
            relu(a)
    """

    def decorate(func):

        def wrapper(*args, **kwargs):
            with ipu_shard_guard(index=index, stage=stage):
                return func(*args, **kwargs)

        return wrapper

    from .dygraph.layers import Layer
    if not isinstance(call_func, Layer):
        if callable(call_func):
            return decorate(call_func)
        else:
            raise TypeError(
                "Unsupported type. Only accept paddle.nn.Layer or function.")

    # patch paddle.nn.Layer
    class BlockFn(type(call_func)):

        def __call__(self, *args, **kwargs):
            with ipu_shard_guard(index=index, stage=stage):
                return super().__call__(*args, **kwargs)

    BlockFn.__name__ = type(call_func).__name__
    call_func.__class__ = BlockFn
    return call_func


389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
def require_version(min_version, max_version=None):
    """
        Check if the installed version of PaddlePaddle is in [min_version, max_version],
        if the installed version is lower than ``min_version`` or higher than ``max_version``,
        an exception will be thrown, NO returns if the installed version is satisfied.

        Args:
            min_version (str): the minimum version required (like '1.4.0').
            max_version (str, optional): the max version required (like '1.6.0'), default is None,
                meaning any version equal or higher than ``min_version`` is acceptable.

        Returns:
            None.

        Raises:
            TypeError: if the type of ``min_version`` is not str.
            TypeError: if the type of ``max_version`` is not str or type(None).
            ValueError: if the value of ``min_version`` is not in version format.
            ValueError: if the value of ``max_version`` is not in version format or None.
            Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                # any version >= 0.1.0 is acceptable.
                fluid.require_version('0.1.0')

                # if 0.1.0 <= version <= 10.0.0, it is acceptable.
                fluid.require_version(min_version='0.1.0', max_version='10.0.0')
        """
    if not isinstance(min_version, str):
        raise TypeError(
            "The type of 'min_version' in require_version must be str, but received %s."
            % (type(min_version)))

    if not isinstance(max_version, (str, type(None))):
        raise TypeError(
            "The type of 'max_version' in require_version must be str or type(None), but received %s."
            % (type(max_version)))

    check_format = re.match(r'\d+(\.\d+){0,3}', min_version)
    if check_format is None or check_format.group() != min_version:
        raise ValueError(
            "The value of 'min_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
            "like '1.5.2.0', but received %s" % min_version)

    if max_version is not None:
        check_format = re.match(r'\d+(\.\d+){0,3}', max_version)
        if check_format is None or check_format.group() != max_version:
            raise ValueError(
                "The value of 'max_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
                "like '1.5.2.0', but received %s" % max_version)

    version_installed = [
        fluid_version.major, fluid_version.minor, fluid_version.patch,
        fluid_version.rc
    ]
    zero_version = ['0', '0', '0', '0']

    def version_cmp(ver_a, ver_b):
        for i in six.moves.range(len(ver_a)):
            if int(ver_a[i]) > int(ver_b[i]):
                return 1
            elif int(ver_a[i]) < int(ver_b[i]):
                return -1
        return 0

    if version_cmp(version_installed, zero_version) == 0:
        if max_version is not None:
            warnings.warn(
                "PaddlePaddle version in [%s, %s] required, but %s installed. "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, max_version, fluid_version.full_version))
        else:
            warnings.warn(
                "PaddlePaddle version %s or higher is required, but %s installed, "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, fluid_version.full_version))
        return

    min_version_split = min_version.split('.')
474 475
    min_version_to_check = min_version_split + zero_version[
        len(min_version_split):]
476 477 478

    if max_version is not None:
        max_version_split = max_version.split('.')
479 480
        max_version_to_check = max_version_split + zero_version[
            len(max_version_split):]
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495

        if version_cmp(version_installed,
                       max_version_to_check) > 0 or version_cmp(
                           version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version in [%s, %s] required, but %s installed."
                % (min_version, max_version, fluid_version.full_version))
    else:
        if version_cmp(version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version %s or higher is required, but %s installed, "
                "please upgrade your PaddlePaddle to %s or other higher version."
                % (min_version, fluid_version.full_version, min_version))


496
def _dygraph_not_support_(func):
497

498
    def __impl__(*args, **kwargs):
J
Jiabin Yang 已提交
499 500
        assert not _non_static_mode(
        ), "We don't support %s in dynamic graph mode" % func.__name__
501 502 503 504 505 506
        return func(*args, **kwargs)

    return __impl__


def _dygraph_only_(func):
507

508
    def __impl__(*args, **kwargs):
J
Jiabin Yang 已提交
509
        assert _non_static_mode(
510 511 512 513 514 515 516
        ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__
        return func(*args, **kwargs)

    return __impl__


def _static_only_(func):
517

518
    def __impl__(*args, **kwargs):
J
Jiabin Yang 已提交
519
        assert not _non_static_mode(
520
        ), "In PaddlePaddle 2.x, we turn on dynamic graph mode by default, and '%s()' is only supported in static graph mode. So if you want to use this api, please call 'paddle.enable_static()' before this api to enter static graph mode." % func.__name__
521 522 523 524 525
        return func(*args, **kwargs)

    return __impl__


526 527 528 529 530
def _set_pipeline_stage(stage):
    global _current_pipeline_stage
    _current_pipeline_stage = stage


531 532 533 534 535 536
# NOTE(zhiqiu): This decorator is used for the APIs of Variable which is only
# used to make Variable and VarBase has same interfaces, like numpy. Since VarBase is not exposed in our
# official docments, logically, we want to keep VarBase and logically consistent. While, actually,
# in our implementation, there some APIs not supported, like numpy, because Variable contains the desc.
# So, those APIs are listed under class Variable to generate docs only.
# TODO(zhiqiu): We should make VarBase consistent with Variable in future, for example, by inheritting
T
tangwei12 已提交
537
# same base class.
538
def _fake_interface_only_(func):
539

540 541
    def __impl__(*args, **kwargs):
        raise AssertionError(
542 543 544 545 546
            "'%s' only can be called by `paddle.Tensor` in dynamic graph mode. Suggestions:\n"
            "  1. If you are in static graph mode, you can switch to dynamic graph mode by turning off `paddle.enable_static()` or calling `paddle.disable_static()`.\n"
            "  2. If you are using `@paddle.jit.to_static`, you can turn off ProgramTranslator by calling `paddle.jit.ProgramTranslator().enable(False)`. "
            "If you have to translate dynamic graph to static graph, please use other API to replace '%s'."
            % (func.__name__, func.__name__))
547 548 549 550

    return __impl__


T
tangwei12 已提交
551 552
# NOTE(chenweihang): There is argument name typo (stat_dict, correct name is state_dict)
# in fluid api Layer.set_dict, Optimizer.load, in order to correct the argument without
553 554 555 556
# introducing compatibility issues, add this decorator
# NOTE(chenweihang): not using `wrap_decorator` here is because `wrap_decorator` will
# move kwargs to args, which doesn't work in this decorate case
def deprecate_stat_dict(func):
557

558 559 560 561 562 563 564 565 566 567 568 569 570
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        if 'stat_dict' in kwargs:
            warnings.warn(
                "The argument `stat_dict` has deprecated, please change it to `state_dict`.",
                DeprecationWarning)
            kwargs['state_dict'] = kwargs['stat_dict']
            kwargs.pop('stat_dict')
        return func(*args, **kwargs)

    return wrapper


571 572
dygraph_not_support = wrap_decorator(_dygraph_not_support_)
dygraph_only = wrap_decorator(_dygraph_only_)
573
static_only = wrap_decorator(_static_only_)
574
fake_interface_only = wrap_decorator(_fake_interface_only_)
575 576


L
lujun 已提交
577 578
def _dygraph_tracer():
    return _dygraph_tracer_
579

W
Wu Yi 已提交
580

581 582 583 584
def _global_flags():
    return _global_flags_


M
minqiyang 已提交
585
def _current_expected_place():
586 587 588
    global _global_expected_place_
    if _global_expected_place_ is None:
        if core.is_compiled_with_cuda():
589 590 591 592 593
            try:
                device_count = core.get_cuda_device_count()
            except Exception as e:
                device_count = 0
            if device_count > 0:
594
                _global_expected_place_ = core.CUDAPlace(_cuda_ids()[0])
595 596 597 598 599
            else:
                warnings.warn(
                    "You are using GPU version Paddle, but your CUDA device is not set properly. CPU device will be used by default."
                )
                _global_expected_place_ = core.CPUPlace()
600 601 602 603 604 605
        elif core.is_compiled_with_xpu():
            try:
                device_count = core.get_xpu_device_count()
            except Exception as e:
                device_count = 0
            if device_count > 0:
606
                _global_expected_place_ = core.XPUPlace(_xpu_ids()[0])
607 608 609 610 611
            else:
                warnings.warn(
                    "You are using XPU version Paddle, but your XPU device is not set properly. CPU device will be used by default."
                )
                _global_expected_place_ = core.CPUPlace()
612 613 614 615 616 617
        elif core.is_compiled_with_mlu():
            try:
                device_count = core.get_mlu_device_count()
            except Exception as e:
                device_count = 0
            if device_count > 0:
618
                _global_expected_place_ = core.MLUPlace(_mlu_ids()[0])
619 620 621 622 623
            else:
                warnings.warn(
                    "You are using MLU version Paddle, but your MLU device is not set properly. CPU device will be used by default."
                )
                _global_expected_place_ = core.CPUPlace()
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
        else:
            _global_expected_place_ = core.CPUPlace()

    return _global_expected_place_


def _set_dygraph_tracer_expected_place(place):
    global _dygraph_tracer_
    if _dygraph_tracer_ is not None:
        _dygraph_tracer_._expected_place = place


def _set_expected_place(place):
    global _global_expected_place_
    _global_expected_place_ = place
J
Jiabin Yang 已提交
639
    _set_dygraph_tracer_expected_place(place)
M
minqiyang 已提交
640 641


L
Leo Chen 已提交
642 643
# TODO(zhiqiu): remove this function.
def _var_base_to_np(var_base):
644 645
    """
    convert VarBase tp numpy
T
tangwei12 已提交
646

647 648 649
    Args:
        var_base(VarBase) : the VarBase to convert
    Returns (np.ndarray): the np.ndarray contain the value of VarBase
L
Leo Chen 已提交
650 651 652 653 654 655 656 657 658
    """

    warnings.warn(
        "paddle.fluid.framework._var_base_to_np is deprecated, please use var_base.numpy() instead of _var_base_to_np(var_base)."
    )

    return var_base.numpy()


S
sneaxiy 已提交
659
def _cpu_num():
660
    if "CPU_NUM" not in os.environ.keys():
C
chengduo 已提交
661 662 663 664 665 666 667 668
        if multiprocessing.cpu_count() > 1:
            sys.stderr.write(
                '!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.\n'
                'CPU_NUM indicates that how many CPUPlace are used in the current task.\n'
                'And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.\n\n'
                'export CPU_NUM={} # for example, set CPU_NUM as number of physical CPU core which is {}.\n\n'
                '!!! The default number of CPU_NUM=1.\n'.format(
                    multiprocessing.cpu_count(), multiprocessing.cpu_count()))
C
chengduo 已提交
669
        os.environ['CPU_NUM'] = str(1)
670
    cpu_num = os.environ.get('CPU_NUM')
C
chengduo 已提交
671 672 673 674 675 676 677 678 679 680
    return int(cpu_num)


def _cuda_ids():
    gpus_env = os.getenv("FLAGS_selected_gpus")
    if gpus_env:
        device_ids = [int(s) for s in gpus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_cuda_device_count())
    return device_ids
S
sneaxiy 已提交
681 682


683 684 685 686 687 688 689 690 691
def _xpu_ids():
    xpus_env = os.getenv("FLAGS_selected_xpus")
    if xpus_env:
        device_ids = [int(s) for s in xpus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_xpu_device_count())
    return device_ids


692 693 694 695 696 697 698 699 700
def _npu_ids():
    npus_env = os.getenv("FLAGS_selected_npus")
    if npus_env:
        device_ids = [int(s) for s in npus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_npu_device_count())
    return device_ids


701 702 703 704 705 706 707 708 709
def _mlu_ids():
    mlus_env = os.getenv("FLAGS_selected_mlus")
    if mlus_env:
        device_ids = [int(s) for s in mlus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_mlu_device_count())
    return device_ids


710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
def is_compiled_with_xpu():
    """
    Whether this whl package can be used to run the model on XPU.

    Returns (bool): support xpu or not.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            support_xpu = fluid.is_compiled_with_xpu()
    """
    return core.is_compiled_with_xpu()


725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
def is_compiled_with_npu():
    """
    Whether this whl package can be used to run the model on NPU.

    Returns (bool): support npu or not.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            support_npu = fluid.is_compiled_with_npu()
    """
    return core.is_compiled_with_npu()


740 741 742 743 744 745 746
def disable_signal_handler():
    """
    Reset signal handler registered by Paddle.

    Paddle installs signal handlers at C++ level to log debug information upon failing.
    However, conflicts can happen if another python module is making use of such signal.
    Such being the case, one may disblae paddle signal handler via this interface.
747

748 749 750 751 752 753
    Known frameworks that require disabling signal handler includes:
    1. TVM
    2. ADLIK

    Make sure you called paddle.disable_signal_handler() before using above mentioned frameworks.

754
    Returns: None
755 756 757 758 759 760 761 762 763 764

    Examples:
        .. code-block:: python

            import paddle
            paddle.disable_signal_handler()
    """
    core.disable_signal_handler()


765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
def is_compiled_with_cinn():
    """
    Whether this whl package can be used to run the model on CINN.

    Returns (bool): `True` if CINN is currently available, otherwise `False`.

    Examples:
        .. code-block:: python

            import paddle
            support_cinn = paddle.device.is_compiled_with_cinn()
    """
    return core.is_compiled_with_cinn()


C
chengduo 已提交
780 781 782 783
def is_compiled_with_cuda():
    """
    Whether this whl package can be used to run the model on GPU.

784
    Returns (bool): `True` if CUDA is currently available, otherwise `False`.
C
chengduo 已提交
785 786 787 788

    Examples:
        .. code-block:: python

789
            import paddle
790
            support_gpu = paddle.device.is_compiled_with_cuda()
C
chengduo 已提交
791 792 793 794
    """
    return core.is_compiled_with_cuda()


795 796 797 798 799 800 801 802 803 804
def is_compiled_with_rocm():
    """
    Whether this whl package can be used to run the model on AMD or Hygon GPU(ROCm).

    Returns (bool): `True` if ROCm is currently available, otherwise `False`.

    Examples:
        .. code-block:: python

            import paddle
805
            support_gpu = paddle.device.is_compiled_with_rocm()
806 807 808 809
    """
    return core.is_compiled_with_rocm()


S
sneaxiy 已提交
810
def cuda_places(device_ids=None):
L
lujun 已提交
811
    """
812
    Note:
813 814 815
        For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device.
        The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable.

C
Chen Weihang 已提交
816
    This function creates a list of :code:`paddle.CUDAPlace` objects.
S
add doc  
sneaxiy 已提交
817 818

    If :code:`device_ids` is None, environment variable of
819
    :code:`FLAGS_selected_gpus` would be checked first. For example, if
S
add doc  
sneaxiy 已提交
820
    :code:`FLAGS_selected_gpus=0,1,2`, the returned list would
C
Chen Weihang 已提交
821
    be [paddle.CUDAPlace(0), paddle.CUDAPlace(1), paddle.CUDAPlace(2)].
S
add doc  
sneaxiy 已提交
822
    If :code:`FLAGS_selected_gpus` is not set, all visible
823
    gpu places would be returned according to the :code:`CUDA_VISIBLE_DEVICES` environment variable.
S
add doc  
sneaxiy 已提交
824 825

    If :code:`device_ids` is not None, it should be the device
826
    ids of GPUs. For example, if :code:`device_ids=[0,1,2]`,
827
    the returned list would be
C
Chen Weihang 已提交
828
    [paddle.CUDAPlace(0), paddle.CUDAPlace(1), paddle.CUDAPlace(2)].
T
tangwei12 已提交
829

830
    Parameters:
831
        device_ids (list|tuple, optional): A list/tuple of int of GPU device ids.
S
add doc  
sneaxiy 已提交
832 833

    Returns:
C
Chen Weihang 已提交
834
        list of paddle.CUDAPlace: Created GPU place list.
L
lujun 已提交
835 836

    Examples:
837

L
lujun 已提交
838 839
        .. code-block:: python

C
Chen Weihang 已提交
840 841
            import paddle
            import paddle.static as static
T
tangwei12 已提交
842

843
            # required: gpu
844

C
Chen Weihang 已提交
845 846 847
            paddle.enable_static()

            cuda_places = static.cuda_places()
L
lujun 已提交
848 849

    """
S
sneaxiy 已提交
850 851 852
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_ids is None:
C
chengduo 已提交
853
        device_ids = _cuda_ids()
S
sneaxiy 已提交
854 855 856 857 858
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.CUDAPlace(dev_id) for dev_id in device_ids]


859 860 861 862
def xpu_places(device_ids=None):
    """
    **Note**:
        For multi-card tasks, please use `FLAGS_selected_xpus` environment variable to set the visible XPU device.
S
sunzhongkai588 已提交
863 864 865 866 867 868 869 870 871
        This function creates a list of :code:`paddle.XPUPlace` objects.
        If :code:`device_ids` is None, environment variable of
        :code:`FLAGS_selected_xpus` would be checked first. For example, if
        :code:`FLAGS_selected_xpus=0,1,2`, the returned list would
        be [paddle.XPUPlace(0), paddle.XPUPlace(1), paddle.XPUPlace(2)].
        If :code:`FLAGS_selected_xpus` is not set, all visible
        xpu places would be returned.
        If :code:`device_ids` is not None, it should be the device
        ids of XPUs. For example, if :code:`device_ids=[0,1,2]`,
872
        the returned list would be
S
sunzhongkai588 已提交
873
        [paddle.XPUPlace(0), paddle.XPUPlace(1), paddle.XPUPlace(2)].
874

875 876 877 878 879 880
    Parameters:
        device_ids (list or tuple of int, optional): list of XPU device ids.
    Returns:
        list of paddle.XPUPlace: Created XPU place list.
    Examples:
        .. code-block:: python
S
sunzhongkai588 已提交
881

882 883
            # required: xpu

884 885
            import paddle
            import paddle.static as static
886

887 888 889 890 891 892 893 894 895 896 897 898
            paddle.enable_static()
            xpu_places = static.xpu_places()
    """
    assert core.is_compiled_with_xpu(), \
        "Not compiled with XPU"
    if device_ids is None:
        device_ids = _xpu_ids()
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.XPUPlace(dev_id) for dev_id in device_ids]


899 900 901 902
def npu_places(device_ids=None):
    """
    **Note**:
        For multi-card tasks, please use `FLAGS_selected_npus` environment variable to set the visible NPU device.
903

904 905 906 907 908 909 910 911 912
    This function creates a list of :code:`paddle.NPUPlace` objects.
    If :code:`device_ids` is None, environment variable of
    :code:`FLAGS_selected_npus` would be checked first. For example, if
    :code:`FLAGS_selected_npus=0,1,2`, the returned list would
    be [paddle.NPUPlace(0), paddle.NPUPlace(1), paddle.NPUPlace(2)].
    If :code:`FLAGS_selected_npus` is not set, all visible
    npu places would be returned.
    If :code:`device_ids` is not None, it should be the device
    ids of NPUs. For example, if :code:`device_ids=[0,1,2]`,
913
    the returned list would be
914
    [paddle.NPUPlace(0), paddle.NPUPlace(1), paddle.NPUPlace(2)].
915

916 917 918 919 920 921 922 923 924 925 926
    Parameters:
        device_ids (list or tuple of int, optional): list of NPU device ids.
    Returns:
        list of paddle.NPUPlace: Created NPU place list.
    Examples:
        .. code-block:: python

            # required: npu

            import paddle
            import paddle.static as static
927

928 929 930 931 932 933 934 935 936 937 938 939
            paddle.enable_static()
            npu_places = static.npu_places()
    """
    assert core.is_compiled_with_npu(), \
        "Not compiled with NPU"
    if device_ids is None:
        device_ids = _npu_ids()
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.NPUPlace(dev_id) for dev_id in device_ids]


S
sneaxiy 已提交
940
def cpu_places(device_count=None):
L
lujun 已提交
941
    """
C
Chen Weihang 已提交
942
    This function creates a list of :code:`paddle.CPUPlace` objects, and returns the created list.
T
tangwei12 已提交
943

S
add doc  
sneaxiy 已提交
944
    If :code:`device_count` is None, the device count would
945
    be determined by environment variable :code:`CPU_NUM`.
C
chengduo 已提交
946 947
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
948 949
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
950

951 952
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
953 954

    Returns:
C
Chen Weihang 已提交
955
        list of paddle.CPUPlace: Created list of CPU places.
L
lujun 已提交
956 957

    Examples:
958

L
lujun 已提交
959 960
        .. code-block:: python

C
Chen Weihang 已提交
961 962
            import paddle
            import paddle.static as static
T
tangwei12 已提交
963

C
Chen Weihang 已提交
964 965 966
            paddle.enable_static()

            cpu_places = static.cpu_places()
L
lujun 已提交
967 968
    """

S
sneaxiy 已提交
969 970 971 972 973 974
    if device_count is None:
        device_count = _cpu_num()
    return [core.CPUPlace()] * device_count


def cuda_pinned_places(device_count=None):
L
lujun 已提交
975
    """
976
    This function creates a list of :code:`fluid.CUDAPinnedPlace` objects.
S
add doc  
sneaxiy 已提交
977 978

    If :code:`device_count` is None, the device count would
979
    be determined by environment variable :code:`CPU_NUM`.
980 981 982 983
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
984

985 986
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
987 988

    Returns:
989
        list of fluid.CUDAPinnedPlace: Created list of CUDA pinned places.
L
lujun 已提交
990 991 992 993

    Examples:
        .. code-block:: python

994
            import paddle.fluid as fluid
L
lujun 已提交
995 996 997 998 999
            cuda_pinned_places_cpu_num = fluid.cuda_pinned_places()
            # or
            cuda_pinned_places = fluid.cuda_pinned_places(1)

    """
S
sneaxiy 已提交
1000 1001 1002
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_count is None:
1003 1004
        device_count = len(_cuda_ids())
    return [core.CUDAPinnedPlace()] * device_count
S
sneaxiy 已提交
1005 1006


1007 1008
def mlu_places(device_ids=None):
    """
G
gouzil 已提交
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
    This function creates a list of :code:`paddle.device.MLUPlace` objects.
    If :code:`device_ids` is None, environment variable of
    :code:`FLAGS_selected_mlus` would be checked first. For example, if
    :code:`FLAGS_selected_mlus=0,1,2`, the returned list would
    be [paddle.device.MLUPlace(0), paddle.device.MLUPlace(1), paddle.device.MLUPlace(2)].
    If :code:`FLAGS_selected_mlus` is not set, all visible
    mlu places would be returned.
    If :code:`device_ids` is not None, it should be the device
    ids of MLUs. For example, if :code:`device_ids=[0,1,2]`,
    the returned list would be
    [paddle.device.MLUPlace(0), paddle.device.MLUPlace(1), paddle.device.MLUPlace(2)].

    Note:
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
        For multi-card tasks, please use `FLAGS_selected_mlus` environment variable to set the visible MLU device.

    Parameters:
        device_ids (list or tuple of int, optional): list of MLU device ids.

    Returns:
        list of paddle.device.MLUPlace: Created MLU place list.

    Examples:
        .. code-block:: python

            # required: mlu

            import paddle
            import paddle.static as static

            paddle.enable_static()
            mlu_places = static.mlu_places()
    """
    assert core.is_compiled_with_mlu(), \
        "Not compiled with MLU"
    if device_ids is None:
        device_ids = _mlu_ids()
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.MLUPlace(dev_id) for dev_id in device_ids]


1050
class NameScope(object):
1051

1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
    def __init__(self, name="", parent=None):
        self._children = dict()
        self._name = name
        self._parent = parent

    def child(self, prefix):
        if prefix not in self._children:
            new_child = NameScope(prefix, self)
            self._children[prefix] = [new_child]
        else:
            new_child = NameScope(prefix + "_%d" % len(self._children[prefix]),
                                  self)
            self._children[prefix].append(new_child)
        return new_child

    def parent(self):
        return self._parent

    def name(self):
        return self._name


_name_scope = NameScope()


S
rename  
sneaxiy 已提交
1077
@signature_safe_contextmanager
1078 1079
def name_scope(prefix=None):
    """
1080

1081
    Generate hierarchical name prefix for the operators in Static Graph.
1082

1083
    Note:
T
Tao Luo 已提交
1084 1085
        This should only used for debugging and visualization purpose.
        Don't use it for serious analysis such as graph/program transformations.
1086
        Don't use it in dygraph, since it will cause memory leak.
1087 1088

    Args:
T
Tao Luo 已提交
1089
        prefix(str, optional): prefix. Default is none.
1090 1091

    Examples:
1092

1093
        .. code-block:: python
T
Tink_Y 已提交
1094

1095 1096 1097
          import paddle
          paddle.enable_static()
          with paddle.static.name_scope("s1"):
1098
             a = paddle.static.data(name='data', shape=[None, 1], dtype='int32')
T
Tao Luo 已提交
1099
             b = a + 1
1100
             with paddle.static.name_scope("s2"):
T
Tao Luo 已提交
1101
                c = b * 1
1102
             with paddle.static.name_scope("s3"):
T
Tao Luo 已提交
1103
                d = c / 1
1104 1105 1106
          with paddle.static.name_scope("s1"):
                f = paddle.tensor.pow(d, 2.0)
          with paddle.static.name_scope("s4"):
T
Tao Luo 已提交
1107 1108
                g = f - 1

1109
          # Op are created in the default main program.
1110
          for op in paddle.static.default_main_program().block(0).ops:
T
Tao Luo 已提交
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
              # elementwise_add is created in /s1/
              if op.type == 'elementwise_add':
                  assert op.desc.attr("op_namescope") == '/s1/'
              # elementwise_mul is created in '/s1/s2'
              elif op.type == 'elementwise_mul':
                  assert op.desc.attr("op_namescope") == '/s1/s2/'
              # elementwise_div is created in '/s1/s3'
              elif op.type == 'elementwise_div':
                  assert op.desc.attr("op_namescope") == '/s1/s3/'
              # elementwise_sum is created in '/s4'
              elif op.type == 'elementwise_sub':
                  assert op.desc.attr("op_namescope") == '/s4/'
              # pow is created in /s1_1/
              elif op.type == 'pow':
                  assert op.desc.attr("op_namescope") == '/s1_1/'
1126 1127
    """
    # TODO(panyx0718): Only [0-9a-z].
1128
    # in dygraph we don't need namescope since it will cause mem leak
J
Jiabin Yang 已提交
1129
    if _non_static_mode():
L
Leo Chen 已提交
1130 1131
        yield
    else:
T
tianshuo78520a 已提交
1132
        assert prefix, "namescope prefix can not be empty."
1133 1134
        global _name_scope
        _name_scope = _name_scope.child(prefix)
1135 1136 1137 1138
        try:
            yield
        finally:
            _name_scope = _name_scope.parent()
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150


def _full_name_scope():
    global _name_scope
    scope = _name_scope
    name = ""
    while scope:
        name = scope.name() + "/" + name
        scope = scope.parent()
    return name


W
Wu Yi 已提交
1151 1152 1153
def generate_control_dev_var_name():
    import random
    return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random())
Q
qiaolongfei 已提交
1154 1155 1156 1157


def grad_var_name(var_name):
    """
1158 1159
    Returns:
        str: gradient name for a certain var name
Q
qiaolongfei 已提交
1160 1161 1162
    """
    return var_name + GRAD_VAR_SUFFIX

Y
Yu Yang 已提交
1163

1164
def convert_np_dtype_to_dtype_(np_dtype):
1165
    """
1166
    Convert the data type in numpy to the data type in Paddle.
1167

1168
    Args:
1169 1170
        np_dtype (np.dtype|str): The data type in numpy or valid data type
            string.
1171

1172
    Returns:
1173
        core.VarDesc.VarType: The data type in Paddle.
1174 1175

    """
1176 1177
    # Convert the data type string to numpy data type.
    if isinstance(np_dtype, str) and np_dtype == "bfloat16":
1178 1179 1180
        dtype = np.uint16
    else:
        dtype = np.dtype(np_dtype)
1181

1182
    if dtype == np.float32:
1183
        return core.VarDesc.VarType.FP32
1184
    elif dtype == np.float64:
1185
        return core.VarDesc.VarType.FP64
1186
    elif dtype == np.float16:
1187
        return core.VarDesc.VarType.FP16
1188
    elif dtype == np.int32:
1189
        return core.VarDesc.VarType.INT32
1190
    elif dtype == np.int16:
1191
        return core.VarDesc.VarType.INT16
1192
    elif dtype == np.int64:
1193
        return core.VarDesc.VarType.INT64
1194
    elif dtype == np.bool_:
1195
        return core.VarDesc.VarType.BOOL
1196
    elif dtype == np.uint16:
1197 1198 1199
        # since there is still no support for bfloat16 in NumPy,
        # uint16 is used for casting bfloat16
        return core.VarDesc.VarType.BF16
1200 1201
    elif dtype == np.uint8:
        return core.VarDesc.VarType.UINT8
Q
qingqing01 已提交
1202 1203
    elif dtype == np.int8:
        return core.VarDesc.VarType.INT8
1204 1205 1206 1207
    elif dtype == np.complex64:
        return core.VarDesc.VarType.COMPLEX64
    elif dtype == np.complex128:
        return core.VarDesc.VarType.COMPLEX128
1208
    else:
M
minqiyang 已提交
1209
        raise ValueError("Not supported numpy dtype %s" % dtype)
1210 1211 1212


def dtype_is_floating(dtype):
1213 1214 1215
    """
    Check the data type is floating or not.
    Args:
1216
        dtype(np.dtype|core.VarDesc.VarType): data type.
1217 1218 1219 1220 1221
            Could be numpy format or Paddle format

    Returns(bool): True if data type is a float value

    """
1222
    if not isinstance(dtype, core.VarDesc.VarType):
1223 1224
        dtype = convert_np_dtype_to_dtype_(dtype)

1225 1226 1227 1228
    return dtype in [
        core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32,
        core.VarDesc.VarType.FP64
    ]
1229 1230


Y
Yang Yang(Tony) 已提交
1231
def _debug_string_(proto, throw_on_error=True):
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
    """
    Get the debug string of a protobuf message. The message could be not
    initialized.
    Args:
        proto(google.protobuf.message.Message): The protobuf message
        throw_on_error(bool): True if raise an error when the protobuf message
            is not initialized.

    Returns(str): The debug string of the protobuf message

    """
Y
Yu Yang 已提交
1243
    error_fields = list()
Y
Yang Yang(Tony) 已提交
1244
    if not proto.IsInitialized(error_fields) and throw_on_error:
1245 1246 1247
        raise ValueError(
            "{0} are not initialized.\nThe message is {1}:\n".format(
                error_fields, proto))
Y
Yu Yang 已提交
1248 1249 1250
    return proto.__str__()


1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR,
                     name=None,
                     shape=None,
                     dtype=None,
                     persistable=None,
                     **kwargs):
    if dtype is not None:
        if not isinstance(dtype, core.VarDesc.VarType):
            dtype = convert_np_dtype_to_dtype_(dtype)

J
Jiabin Yang 已提交
1261
    if _in_eager_mode_:
1262
        eager_tensor = core.eager.Tensor(
1263
            dtype if dtype else core.VarDesc.VarType.FP32,
1264 1265 1266
            list(shape) if shape else [], name,
            type if type else core.VarDesc.VarType.LOD_TENSOR,
            True if persistable else False)
1267 1268
        eager_tensor.retain_grads()
        return eager_tensor
J
Jiabin Yang 已提交
1269 1270
    else:
        return core.VarBase(dtype if dtype else core.VarDesc.VarType.FP32,
1271 1272 1273
                            list(shape) if shape else [], name,
                            type if type else core.VarDesc.VarType.LOD_TENSOR,
                            True if persistable else False)
1274 1275


1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
def _all_is_type(vals, expected_type):
    """
    Return True if type of each element is expected_type.

    NOTE: BuiltIn all() will always return True if vals is empty.
    """
    assert isinstance(vals, (list, tuple))
    if not vals: return False
    return all(isinstance(v, expected_type) for v in vals)


1287
class VariableMetaClass(type):
1288

1289 1290 1291 1292
    @classmethod
    def __instancecheck__(cls, instance):
        t = type(instance)
        if in_dygraph_mode():
J
Jiabin Yang 已提交
1293
            return issubclass(t, core.eager.Tensor)
1294
        else:
J
Jiabin Yang 已提交
1295 1296
            if _in_legacy_dygraph():
                return issubclass(t, core.VarBase)
1297 1298 1299 1300
            return issubclass(t, Variable)


class ParameterMetaClass(VariableMetaClass):
1301

1302 1303 1304 1305
    @classmethod
    def __instancecheck__(cls, instance):
        t = type(instance)
        if in_dygraph_mode():
J
Jiabin Yang 已提交
1306
            return issubclass(t, EagerParamBase)
1307
        else:
J
Jiabin Yang 已提交
1308 1309
            if _in_legacy_dygraph():
                return issubclass(t, ParamBase)
1310 1311 1312 1313
            return issubclass(t, Parameter)


@six.add_metaclass(VariableMetaClass)
X
Xin Pan 已提交
1314
class Variable(object):
1315
    """
J
Jiabin Yang 已提交
1316
    **Notes**:
1317
        **The constructor of Variable should not be invoked directly.**
J
Jiabin Yang 已提交
1318

1319 1320
        **In Static Graph Mode: Please use** `Block.create_var` **to create a Static variable which has no data until being feed.**

J
Jiabin Yang 已提交
1321 1322 1323
        **In Dygraph Mode: Please use** :ref:`api_fluid_dygraph_to_variable` **to create a dygraph variable with real data**

    In Fluid, every input and output of an OP is a variable. In most
1324
    cases, variables are used for holding different kinds of data or training
J
Jiabin Yang 已提交
1325 1326
    labels. A variable belongs to a :ref:`api_guide_Block_en` . All variable has its own name and
    two variables in different :ref:`api_guide_Block_en` could have the same name.
1327

1328
    There are many kinds of variables. Each kind of them has its own attributes
J
Jiabin Yang 已提交
1329
    and usages. Please refer to the `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_ for details.
1330

T
tianshuo78520a 已提交
1331
    Most of a Variable's member variables can be set to be None. It mean
1332
    it is not available or will be specified later.
1333

1334
    Examples:
1335 1336
        In Static Graph Mode:

1337 1338
        .. code-block:: python

1339
            import paddle.fluid as fluid
1340
            cur_program = fluid.Program()
1341 1342 1343 1344
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
S
sunzhongkai588 已提交
1345

J
Jiabin Yang 已提交
1346
        In `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_  Mode:
1347 1348 1349 1350 1351 1352 1353 1354 1355

        .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                new_variable = fluid.dygraph.to_variable(np.arange(10))

1356 1357
    """

Y
Yu Yang 已提交
1358 1359
    def __init__(self,
                 block,
Y
Yu Yang 已提交
1360
                 type=core.VarDesc.VarType.LOD_TENSOR,
Y
Yu Yang 已提交
1361 1362 1363 1364
                 name=None,
                 shape=None,
                 dtype=None,
                 lod_level=None,
1365
                 capacity=None,
Q
QI JUN 已提交
1366
                 persistable=None,
F
fengjiayi 已提交
1367
                 error_clip=None,
Y
Yu Yang 已提交
1368
                 stop_gradient=False,
F
fengjiayi 已提交
1369
                 is_data=False,
H
Huihuang Zheng 已提交
1370
                 need_check_feed=False,
H
hong 已提交
1371
                 belong_to_optimizer=False,
Y
Yu Yang 已提交
1372
                 **kwargs):
Y
Yu Yang 已提交
1373 1374
        self.block = block
        if name is None:
Y
Yu Yang 已提交
1375
            name = unique_name.generate('_generated_var')
D
Dong Zhihong 已提交
1376

Y
Yu Yang 已提交
1377
        if dtype is not None:
1378
            if not isinstance(dtype, core.VarDesc.VarType):
1379
                dtype = convert_np_dtype_to_dtype_(dtype)
1380

S
Steffy-zxf 已提交
1381 1382 1383 1384
        if dtype == core.VarDesc.VarType.STRINGS:
            type = core.VarDesc.VarType.STRINGS
            lod_level = None

1385 1386 1387
        if type == core.VarDesc.VarType.SPARSE_COO:
            lod_level = None

H
hong 已提交
1388 1389
        self.belong_to_optimizer = belong_to_optimizer

1390 1391 1392 1393
        self.error_clip = error_clip

        is_new_var = False
        name = cpt.to_text(name)
1394
        self.desc = self.block.desc.find_var(name.encode())
1395

1396
        if self.desc is None:
1397
            self.desc = self.block.desc.var(name.encode())
1398
            is_new_var = True
1399

1400 1401 1402
        if is_new_var:
            self.desc.set_type(type)
        elif self.desc.type() != type:
L
Leo Chen 已提交
1403 1404
            raise ValueError("Variable '{0}' has been created before. The "
                             "previous type is {1}, the new type is {2}. They"
1405 1406
                             " are not matched".format(self.name,
                                                       self.desc.type(), type))
1407

1408
        if shape is not None:
1409
            if is_new_var:
1410 1411 1412 1413 1414 1415
                self.desc.set_shape(shape)
            else:
                old_shape = self.shape
                shape = tuple(shape)
                if shape != old_shape:
                    raise ValueError(
L
Leo Chen 已提交
1416 1417
                        "Variable '{0}' has been created before. The previous "
                        "shape is {1}, the new shape is {2}. They are not "
1418 1419 1420 1421 1422 1423 1424
                        "matched.".format(self.name, old_shape, shape))
        if dtype is not None:
            if is_new_var:
                self.desc.set_dtype(dtype)
            else:
                old_dtype = self.dtype
                if dtype != old_dtype:
L
Leo Chen 已提交
1425 1426
                    raise ValueError("Variable '{0}' has been created before. "
                                     "The previous data type is {1}, the new "
1427 1428 1429 1430 1431 1432 1433 1434 1435
                                     "data type is {2}. They are not "
                                     "matched.".format(self.name, old_dtype,
                                                       dtype))

        if lod_level is not None:
            if is_new_var:
                self.desc.set_lod_level(lod_level)
            else:
                if lod_level != self.lod_level:
L
Leo Chen 已提交
1436 1437
                    raise ValueError("Variable '{0}' has been created before. "
                                     "The previous lod_level is {1}, the new "
1438 1439 1440 1441 1442 1443 1444 1445 1446
                                     "lod_level is {2}. They are not "
                                     "matched".format(self.name, self.lod_level,
                                                      lod_level))
        if persistable is not None:
            if is_new_var:
                self.desc.set_persistable(persistable)
            else:
                if persistable != self.persistable:
                    raise ValueError(
L
Leo Chen 已提交
1447 1448
                        "Variable '{0}' has been created before."
                        "The previous persistable is {1}, the new "
1449 1450
                        "persistable is {2}. They are not matched".format(
                            self.name, self.persistable, persistable))
1451

1452 1453
        if need_check_feed and is_new_var:
            self.desc.set_need_check_feed(need_check_feed)
H
Huihuang Zheng 已提交
1454

1455 1456 1457 1458 1459 1460 1461
        if capacity is not None:
            if is_new_var:
                self.desc.set_capacity(capacity)
            else:
                # TODO(abhinavarora) : Compare with set capacity once,
                # get_capacity is implemented
                pass
1462

1463 1464
        self.block.vars[name] = self
        self.op = None
1465
        self.stop_gradient = stop_gradient
1466
        self.is_data = is_data
Y
Yu Yang 已提交
1467

1468 1469 1470
    def detach(self):
        """
        Returns a new Variable, detached from the current graph.
1471 1472
        It will share data with origin Variable and without tensor copy.
        In addition, the detached Variable doesn't provide gradient propagation.
1473

1474
        Returns:
J
Jiabin Yang 已提交
1475
             ( :ref:`api_guide_Variable_en` | dtype is same as current Variable): The detached Variable.
1476 1477 1478 1479

        Examples:
            .. code-block:: python

1480
                import paddle
1481

1482 1483 1484 1485
                paddle.enable_static()

                # create a static Variable
                x = paddle.static.data(name='x', shape=[3, 2, 1])
1486

1487 1488
                # create a detached Variable
                y = x.detach()
1489
        """
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501

        assert self.type == core.VarDesc.VarType.SELECTED_ROWS or \
            self.type == core.VarDesc.VarType.LOD_TENSOR, \
            "only support a variable with SELECTED_ROWS or LOD_TENSOR to be detached"

        output = self.block.create_var(
            name=unique_name.generate_with_ignorable_key("detach_" + self.name),
            dtype=self.dtype,
            type=self.type,
            persistable=self.persistable,
            stop_gradient=True)

1502 1503 1504
        self.block.append_op(type='share_data',
                             inputs={'X': [self]},
                             outputs={'Out': [output]})
1505
        return output
1506

1507
    @fake_interface_only
1508
    def numpy(self):
1509
        """
J
Jiabin Yang 已提交
1510
        **Notes**:
T
tianshuo78520a 已提交
1511
            **This API is ONLY available in Dygraph mode**
1512

J
Jiabin Yang 已提交
1513
        Returns a numpy array shows the value of current :ref:`api_guide_Variable_en`
1514 1515 1516 1517 1518

        Returns:
            ndarray: The numpy value of current Variable.

        Returns type:
J
Jiabin Yang 已提交
1519
            ndarray: dtype is same as current Variable
1520 1521 1522 1523 1524 1525

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
1526
                from paddle.fluid.dygraph import Linear
1527 1528 1529 1530
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
1531
                    linear = Linear(32, 64)
1532
                    data = to_variable(data)
1533
                    x = linear(data)
1534 1535 1536
                    print(x.numpy())

        """
1537
        pass
1538

1539
    @fake_interface_only
1540
    def backward(self, retain_graph=False):
1541
        """
J
Jiabin Yang 已提交
1542
        **Notes**:
T
tianshuo78520a 已提交
1543
            **This API is ONLY available in Dygraph mode**
1544

1545
        Run backward of current Graph which starts from current Tensor.
1546

J
Jiabin Yang 已提交
1547
        Args:
1548 1549 1550 1551
            retain_graph(bool, optional): If False, the graph used to compute grads will be freed. If you would
                like to add more ops to the built graph after calling this method( :code:`backward` ), set the parameter
                :code:`retain_graph` to True, then the grads will be retained. Thus, seting it to False is much more memory-efficient.
                Defaults to False.
1552

J
Jiabin Yang 已提交
1553 1554
        Returns:
            NoneType: None
1555 1556 1557 1558 1559

        Examples:
            .. code-block:: python

                import numpy as np
1560 1561
                import paddle
                paddle.disable_static()
1562 1563

                x = np.ones([2, 2], np.float32)
1564 1565 1566 1567 1568 1569 1570
                inputs = []
                for _ in range(10):
                    tmp = paddle.to_tensor(x)
                    # if we don't set tmp's stop_gradient as False then, all path to loss will has no gradient since
                    # there is no one need gradient on it.
                    tmp.stop_gradient=False
                    inputs.append(tmp)
1571 1572
                ret = paddle.add_n(inputs)
                loss = paddle.sum(ret)
1573
                loss.backward()
1574 1575

        """
1576
        pass
1577

1578
    @fake_interface_only
1579
    def gradient(self):
1580
        """
J
Jiabin Yang 已提交
1581
        **Notes**:
T
tianshuo78520a 已提交
1582
            **This API is ONLY available in Dygraph mode**
1583 1584 1585

        Get the Gradient of Current Variable

J
Jiabin Yang 已提交
1586
        Returns:
1587
            ndarray or tuple of ndarray: if Variable's type is LoDTensor, return numpy value of the gradient of current Variable, if Variable's type is SelectedRows, return tuple of ndarray, first element of tuple is numpy value of the gradient of current Variable, second element of tuple is numpy value of the rows of current Variable.
1588 1589 1590 1591 1592 1593 1594

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

1595
                # example1: return ndarray
1596 1597 1598 1599 1600 1601 1602 1603 1604
                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
1605
                    loss2.backward()
1606 1607
                    print(loss2.gradient())

1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
                # example2: return tuple of ndarray
                with fluid.dygraph.guard():
                    embedding = fluid.dygraph.Embedding(
                        size=[20, 32],
                        param_attr='emb.w',
                        is_sparse=True)
                    x_data = np.arange(12).reshape(4, 3).astype('int64')
                    x_data = x_data.reshape((-1, 3, 1))
                    x = fluid.dygraph.base.to_variable(x_data)
                    out = embedding(x)
                    out.backward()
                    print(embedding.weight.gradient())

1621
        """
1622
        pass
1623

1624
    @fake_interface_only
1625
    def clear_gradient(self):
1626
        """
J
Jiabin Yang 已提交
1627
        **Notes**:
T
tianshuo78520a 已提交
1628
            **1. This API is ONLY available in Dygraph mode**
J
Jiabin Yang 已提交
1629 1630

            **2. Use it only Variable has gradient, normally we use this for Parameters since other temporal Variable will be deleted by Python's GC**
1631

J
Jiabin Yang 已提交
1632
        Clear  (set to ``0`` ) the Gradient of Current Variable
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650

        Returns:  None

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
1651
                    loss2.backward()
1652 1653 1654 1655 1656
                    print(loss2.gradient())
                    loss2.clear_gradient()
                    print("After clear {}".format(loss2.gradient()))

        """
1657
        pass
X
Xin Pan 已提交
1658

1659 1660 1661 1662
    @fake_interface_only
    def register_hook(self, hook):
        pass

1663
    def __str__(self):
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
        return self._to_readable_code()

    def _to_readable_code(self):
        """
        Get readable debug string of Variable.

        .. note::
            If you want to get the debug string in protobuf format,
            please use :code:`to_string` method.

        Returns:
            string: The formatted Variable string.

        Examples:
            .. code-block:: python

1680 1681
                import paddle
                import paddle.static as static
1682

1683 1684 1685
                paddle.enable_static()

                cur_program = static.Program()
1686 1687 1688 1689 1690 1691
                cur_block = cur_program.current_block()
                new_variable = cur_block.create_var(name="X",
                                                    shape=[-1, 23, 48],
                                                    dtype='float32')
                print(new_variable._to_readable_code())
        """
1692 1693
        # VarType.LOD_TENSOR -> LOD_TENSOR
        type_str = str(self.type).split('.')[1]
1694
        if self.type == core.VarDesc.VarType.SELECTED_ROWS or self.type == core.VarDesc.VarType.LOD_TENSOR:
1695 1696
            dtype_str = str(self.dtype).split('.')[1]
            var_str = "{name} : {type}.shape{shape}.dtype({dtype}).stop_gradient({stop_gradient})".\
T
tangwei12 已提交
1697 1698
                format(name=self.name, type=type_str, shape=self.shape,
                       dtype=dtype_str, stop_gradient=self.stop_gradient)
1699
        else:
1700 1701
            var_str = "{name} : {type})".\
                format(name=self.name, type=type_str)
1702

1703
        if self.is_parameter:
1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
            if self.trainable:
                var_str = "trainable param " + var_str
            else:
                var_str = "param " + var_str
        else:
            var_str = "var " + var_str

        if self.persistable:
            var_str = "persist " + var_str

1714
        from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context
1715
        dist_context = get_default_distributed_context()
1716 1717
        dist_tensor = dist_context.get_dist_tensor_for_program(self)
        if dist_tensor is not None:
1718 1719
            var_str += ", {name} = {value}".format(name="dist_attr",
                                                   value=dist_tensor)
1720

1721
        return var_str
Y
Yang Yang(Tony) 已提交
1722

F
update  
fengjiayi 已提交
1723
    def to_string(self, throw_on_error, with_details=False):
1724 1725 1726
        """
        Get debug string.

J
Jiabin Yang 已提交
1727 1728 1729 1730 1731
        Args:

            throw_on_error (bool): True if raise an exception when self is not initialized.

            with_details (bool): more details about variables and parameters (e.g. trainable, optimize_attr, ...) will be printed when with_details is True. Default value is False;
1732

1733 1734
        Returns:
            str: The debug string.
1735 1736 1737 1738 1739

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
1740
                import paddle
1741

1742
                paddle.enable_static()
1743 1744 1745 1746 1747
                cur_program = fluid.Program()
                cur_block = cur_program.current_block()
                new_variable = cur_block.create_var(name="X",
                                                    shape=[-1, 23, 48],
                                                    dtype='float32')
1748
                print(new_variable.to_string(True))
J
Jiabin Yang 已提交
1749
                print("=============with detail===============")
1750
                print(new_variable.to_string(True, True))
1751
        """
1752 1753
        assert isinstance(throw_on_error, bool) and isinstance(
            with_details, bool)
1754
        protostr = self.desc.serialize_to_string()
1755
        proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr))
F
update  
fengjiayi 已提交
1756 1757
        res_str = _debug_string_(proto, throw_on_error)
        if with_details:
1758
            additional_attr = ("error_clip", )
F
update  
fengjiayi 已提交
1759
            for attr_name in additional_attr:
1760 1761 1762
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))

F
update  
fengjiayi 已提交
1763
        return res_str
1764 1765 1766

    __repr__ = __str__

1767 1768 1769
    def element_size(self):
        """
        Returns the size in bytes of an element in the Tensor.
1770

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
        Examples:
          .. code-block:: python

            import paddle
            paddle.enable_static()

            x = paddle.static.data(name='x1', shape=[3, 2], dtype='bool')
            x.element_size() # 1

            x = paddle.static.data(name='x2', shape=[3, 2], dtype='int16')
            x.element_size() # 2

            x = paddle.static.data(name='x3', shape=[3, 2], dtype='float16')
            x.element_size() # 2

            x = paddle.static.data(name='x4', shape=[3, 2], dtype='float32')
            x.element_size() # 4

            x = paddle.static.data(name='x5', shape=[3, 2], dtype='float64')
            x.element_size() # 8
        """
        return self.desc.element_size()

1794
    @property
1795
    def stop_gradient(self):
J
Jiabin Yang 已提交
1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
        """
        Indicating if we stop gradient from current Variable

        **Notes: This Property has default value as** ``True`` **in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, while Parameter's default value is False. However, in Static Graph Mode all Variable's default stop_gradient value is** ``False``

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                value0 = np.arange(26).reshape(2, 13).astype("float32")
                value1 = np.arange(6).reshape(2, 3).astype("float32")
                value2 = np.arange(10).reshape(2, 5).astype("float32")
1811 1812
                linear = fluid.Linear(13, 5, dtype="float32")
                linear2 = fluid.Linear(3, 3, dtype="float32")
J
Jiabin Yang 已提交
1813 1814 1815
                a = fluid.dygraph.to_variable(value0)
                b = fluid.dygraph.to_variable(value1)
                c = fluid.dygraph.to_variable(value2)
1816 1817
                out1 = linear(a)
                out2 = linear2(b)
J
Jiabin Yang 已提交
1818 1819 1820 1821
                out1.stop_gradient = True
                out = fluid.layers.concat(input=[out1, out2, c], axis=1)
                out.backward()

1822
                assert linear.weight.gradient() is None
J
Jiabin Yang 已提交
1823 1824
                assert (out1.gradient() == 0).all()
        """
1825
        return self.desc.stop_gradient()
1826

1827 1828
    @stop_gradient.setter
    def stop_gradient(self, s):
1829
        self.desc.set_stop_gradient(s)
1830

1831 1832
    @property
    def persistable(self):
J
Jiabin Yang 已提交
1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
        """
        Indicating if we current Variable should be long-term alive


        **Notes: This Property will be deprecated and this API is just to help user understand concept**

            **1. All Variable's persistable is** ``False`` **except Parameters.**

            **2. In** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, this property should not be changed**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("persistable of current Var is: {}".format(new_variable.persistable))
        """
1854
        return self.desc.persistable()
1855

Y
Yu Yang 已提交
1856 1857
    @persistable.setter
    def persistable(self, p):
1858
        self.desc.set_persistable(p)
Y
Yu Yang 已提交
1859

1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
    @property
    def is_parameter(self):
        """
        Indicating if current Variable is a Parameter

        Examples:
          .. code-block:: python

            import paddle
            new_parameter = paddle.static.create_parameter(name="X",
                                                shape=[10, 23, 48],
                                                dtype='float32')
            if new_parameter.is_parameter:
                print("Current var is a Parameter")
            else:
                print("Current var is not a Parameter")

            # Current var is a Parameter
        """
        return self.desc.is_parameter()

    @is_parameter.setter
    def is_parameter(self, p):
        self.desc.set_is_parameter(p)

Y
Yu Yang 已提交
1885 1886
    @property
    def name(self):
J
Jiabin Yang 已提交
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
        """
        Indicating name of current Variable

        **Notes: If it has two or more Varaible share the same name in the same** :ref:`api_guide_Block_en` **, it means these Variable will share content in no-** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode. This is how we achieve Parameter sharing**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("name of current Var is: {}".format(new_variable.name))
        """
1903
        return cpt.to_text(self.desc.name())
Y
Yu Yang 已提交
1904

1905 1906 1907 1908 1909 1910
    @property
    def grad_name(self):
        """
        Indicating name of the gradient Variable of current Variable.

        **Notes: This is a read-only property. It simply returns name of
S
sunzhongkai588 已提交
1911 1912
        gradient Variable from a naming convention but doesn't guarantee
        the gradient exists.**
T
tangwei12 已提交
1913

1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
        Examples:
          .. code-block:: python

          import paddle.fluid as fluid

          x = fluid.data(name="x", shape=[-1, 23, 48], dtype='float32')
          print(x.grad_name) # output is "x@GRAD"

        """
        return self.name + "@GRAD"

T
typhoonzero 已提交
1925 1926
    @name.setter
    def name(self, new_name):
1927
        self.desc.set_name(new_name)
T
typhoonzero 已提交
1928

Y
Yu Yang 已提交
1929 1930
    @property
    def shape(self):
J
Jiabin Yang 已提交
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947
        """
        Indicating shape of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("shape of current Var is: {}".format(new_variable.shape))

        """
Y
Yu Yang 已提交
1948
        # convert to tuple, make it as same as numpy API.
1949
        return tuple(self.desc.shape())
Y
Yu Yang 已提交
1950 1951

    @property
F
fengjiayi 已提交
1952
    def dtype(self):
J
Jiabin Yang 已提交
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
        """
        Indicating data type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Dtype of current Var is: {}".format(new_variable.dtype))
        """
1969
        return self.desc.dtype()
Y
Yu Yang 已提交
1970 1971 1972

    @property
    def lod_level(self):
J
Jiabin Yang 已提交
1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985
        """
        Indicating ``LoD`` info of current Variable, please refer to  :ref:`api_fluid_LoDTensor_en` to check the meaning
        of ``LoD``

        **Notes**:

            **1. This is a read-only property**

            **2. Don't support this property in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, it's value should be** ``0(int)``

        Examples:
          .. code-block:: python

1986
            import paddle
J
Jiabin Yang 已提交
1987
            import paddle.fluid as fluid
1988 1989

            paddle.enable_static()
J
Jiabin Yang 已提交
1990 1991 1992 1993 1994 1995 1996
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("LoD Level of current Var is: {}".format(new_variable.lod_level))
        """
1997 1998
        if self.type == core.VarDesc.VarType.SELECTED_ROWS:
            raise Exception("SelectedRows DO NOT supprt lod")
1999 2000
        if self.type == core.VarDesc.VarType.STRINGS:
            return None
2001
        return self.desc.lod_level()
Y
Yu Yang 已提交
2002

Y
Yu Yang 已提交
2003 2004
    @property
    def type(self):
J
Jiabin Yang 已提交
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020
        """
        Indicating Type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Type of current Var is: {}".format(new_variable.type))
        """
2021
        return self.desc.type()
Y
Yu Yang 已提交
2022

2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
    @property
    def T(self):
        """
        Permute current Variable with its dimensions reversed.

        If `n` is the dimensions of `x` , `x.T` is equivalent to `x.transpose([n-1, n-2, ..., 0])`.

        Examples:

            .. code-block:: python

                import paddle
                paddle.enable_static()

                x = paddle.ones(shape=[2, 3, 5])
                x_T = x.T

                exe = paddle.static.Executor()
                x_T_np = exe.run(paddle.static.default_main_program(), fetch_list=[x_T])[0]
                print(x_T_np.shape)
                # (5, 3, 2)
        """
        if len(self.shape) == 1:
            return self
        perm = []
        for i in range(len(self.shape)):
            perm.insert(0, i)

        out = self.block.create_var(
            name=unique_name.generate_with_ignorable_key(self.name + '.tmp'),
            dtype=self.dtype,
            type=self.type,
            persistable=False,
            stop_gradient=False)
        input_shape = self.block.create_var(
            name=unique_name.generate_with_ignorable_key(self.name + '.tmp'),
            dtype=self.dtype,
            type=core.VarDesc.VarType.LOD_TENSOR,
            persistable=False,
            stop_gradient=False)

2064 2065 2066 2067 2068 2069 2070
        self.block.append_op(type='transpose2',
                             inputs={'X': [self]},
                             outputs={
                                 'Out': [out],
                                 'XShape': [input_shape]
                             },
                             attrs={'axis': perm})
2071 2072
        return out

2073 2074 2075
    def clone(self):
        """
        Returns a new static Variable, which is the clone of the original static
2076
        Variable. It remains in the current graph, that is, the cloned Variable
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102
        provides gradient propagation. Calling ``out = tensor.clone()`` is same
        as ``out = assign(tensor)`` .

        Returns:
            Variable: The cloned Variable.

        Examples:
            .. code-block:: python

                import paddle

                paddle.enable_static()

                # create a static Variable
                x = paddle.static.data(name='x', shape=[3, 2, 1])
                # create a cloned Variable
                y = x.clone()

        """
        output = self.block.create_var(
            name=unique_name.generate_with_ignorable_key(self.name + "_clone"),
            dtype=self.dtype,
            type=self.type,
            persistable=self.persistable,
            stop_gradient=self.stop_gradient)

2103 2104 2105
        self.block.append_op(type='assign',
                             inputs={'X': [self]},
                             outputs={'Out': [output]})
2106 2107
        return output

W
Wu Yi 已提交
2108
    def _set_error_clip(self, error_clip):
2109 2110 2111 2112 2113 2114 2115 2116 2117
        """
        Set the error_clip.

        Args:
            error_clip(BaseErrorClipAttr) : The new error_clip.

        Returns:
            None
        """
2118 2119
        self.error_clip = error_clip

2120 2121 2122 2123 2124 2125 2126 2127
    def _set_info(self, key, value):
        """
        Set key-value information for this variable.

        Args:
            key(str): Key for this information.
            value(object): The value associated to the key.

2128
        Returns:
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
            None
        """
        if not hasattr(self, "_info"):
            self._info = {}
        self._info[key] = value

    def _get_info(self, key):
        """
        Get the information of this variable corresponding to key.

        Args:
            key(str): Key for this information.

2142
        Returns:
2143 2144 2145 2146 2147 2148
            object
        """
        if hasattr(self, "_info") and key in self._info:
            return self._info[key]
        return None

2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
    def _slice_indices(self, slice, length):
        """
        Reference implementation for the slice.indices method.
        """
        # Compute step and length as integers.
        step = 1 if slice.step is None else slice.step

        # Raise ValueError for negative length or zero step.
        if length < 0:
            raise ValueError("length should not be negative")
        if step == 0:
T
tianshuo78520a 已提交
2160
            raise ValueError("slice step can not be zero")
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170

        # Find lower and upper bounds for start and stop.
        lower = -1 if step < 0 else 0
        upper = length - 1 if step < 0 else length

        # Compute start.
        if slice.start is None:
            start = upper if step < 0 else lower
        else:
            start = slice.start
2171 2172
            start = max(start +
                        length, lower) if start < 0 else min(start, upper)
2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235

        # Compute stop.
        if slice.stop is None:
            stop = lower if step < 0 else upper
        else:
            stop = slice.stop
            stop = max(stop + length, lower) if stop < 0 else min(stop, upper)

        return start, stop, step

    def _detectEllipsis(self, item):
        has_ellipsis = False
        start = 0
        end = len(self.shape)
        for index, o in enumerate(item):
            if o is Ellipsis:
                if has_ellipsis:
                    raise ValueError("Index can have one ellipsis only.")
                has_ellipsis = True
                start = index
            else:
                if has_ellipsis:
                    end = index
        return has_ellipsis, start, end

    def _reconstructSliceinfo(self, item):
        has_ellipsis, start, end = self._detectEllipsis(item)
        if has_ellipsis:
            newitem = []
            for i in range(start):
                newitem.append(item[i])
            for i in range(start, end):
                newitem.append(slice(None, None, None))
            for i in range(end, len(item)):
                newitem.append(item[i])
            return newitem
        else:
            return None

    def _detectContinuesSlice(self, item):
        starts = []
        ends = []
        for index, o in enumerate(item):
            if isinstance(o, int):
                start = int(o)
                if (index > 0 and index >= self.shape[index]) \
                        or (index < 0 and (index + self.shape[index]) < 0):
                    raise IndexError("invalid index")
                start = max(start + self.shape[index], 0) if start < 0 else min(
                    start, self.shape[index])
                starts.append(start)
                ends.append(start + 1)
            elif isinstance(o, slice):
                start, stop, step = self._slice_indices(o, self.shape[index])
                if step == 1 or step == -1:
                    starts.append(start)
                    ends.append(stop)
                else:
                    return False, None
            else:
                raise IndexError("Valid index accept int or slice or ellipsis")
        return True, [starts, ends]

L
lujun 已提交
2236
    def _cloneVar(self, copy=False):
2237 2238
        if not copy:
            return self.block.create_var(
H
Hongyu Liu 已提交
2239 2240
                name=unique_name.generate_with_ignorable_key(self.name),
                dtype=self.dtype)
2241 2242 2243 2244
        else:
            return self

    def _sliceVar(self, axes, starts, ends):
L
lujun 已提交
2245
        new_var = self._cloneVar()
2246 2247 2248 2249 2250 2251 2252 2253
        self.block.append_op(type="slice",
                             inputs={'Input': [self]},
                             outputs={'Out': [new_var]},
                             attrs={
                                 'axes': axes,
                                 'starts': starts,
                                 'ends': ends
                             })
2254 2255 2256
        return new_var

    def _concatVar(self, inputs, axis):
L
lujun 已提交
2257
        new_var = self._cloneVar()
2258 2259 2260 2261 2262 2263
        self.block.append_op(type="concat",
                             inputs={'X': inputs},
                             outputs={'Out': [new_var]},
                             attrs={
                                 'axis': axis,
                             })
2264 2265 2266 2267 2268
        return new_var

    def _sliceAndConcatVar(self, item, axis):
        if isinstance(item, slice):
            if self.shape[axis] < 0:
L
lujun 已提交
2269
                return self._cloneVar(True)
2270 2271 2272 2273 2274 2275 2276
            start, stop, step = self._slice_indices(item, self.shape[axis])
            if step == 1:
                return self._sliceVar([axis], [start], [stop])
            else:
                vars = []
                if step > 0:
                    while start < stop:
2277 2278
                        vars.append(self._sliceVar([axis], [start],
                                                   [start + 1]))
2279 2280 2281
                        start += step
                else:
                    while start > stop:
2282 2283
                        vars.append(self._sliceVar([axis], [start],
                                                   [start + 1]))
2284 2285 2286 2287
                        start += step
                return self._concatVar(vars, axis)
        elif isinstance(item, int):
            if self.shape[axis] < 0:
L
lujun 已提交
2288
                return self._cloneVar(True)
2289
            index = int(item)
2290
            if (index > 0 and index >= self.shape[axis]) \
2291 2292 2293 2294 2295 2296 2297
                    or (index < 0 and (index + self.shape[axis]) < 0):
                raise IndexError("invalid index")
            return self._sliceVar([axis], [index], [index + 1])
        else:
            raise IndexError("Valid index accept int or slice or tuple")

    def __getitem__(self, item):
2298
        return _getitem_impl_(self, item)
2299

2300
    def __setitem__(self, item, value):
2301
        return _setitem_impl_(self, item, value)
2302

2303 2304
    def get_value(self, scope=None):
        """
2305
        Get the value of variable in given scope.
2306 2307

        Args:
2308
            scope(Scope, optional) : If `scope` is None, it will be set to global scope
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
                obtained through 'paddle.static.global_scope()'. Otherwise, use `scope`.
                Default: None

        Returns:
            Tensor: the value in given scope.

        Examples:
            .. code-block:: python

                import paddle
2319
                import paddle.static as static
2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343
                import numpy as np

                paddle.enable_static()

                x = static.data(name="x", shape=[10, 10], dtype='float32')

                y = static.nn.fc(x, 10, name='fc')
                place = paddle.CPUPlace()
                exe = static.Executor(place)
                prog = paddle.static.default_main_program()
                exe.run(static.default_startup_program())
                inputs = np.ones((10, 10), dtype='float32')
                exe.run(prog, feed={'x': inputs}, fetch_list=[y, ])
                path = 'temp/tensor_'
                for var in prog.list_vars():
                    if var.persistable:
                        t = var.get_value()
                        paddle.save(t, path+var.name+'.pdtensor')

                for var in prog.list_vars():
                    if var.persistable:
                        t_load = paddle.load(path+var.name+'.pdtensor')
                        var.set_value(t_load)
        """
2344 2345
        # The 'framework' is a low-level module, and 'executor'
        # can not be imported at the begainning of this file.
2346 2347 2348 2349
        # Therefore, the above two modules are dynamically imported.
        from .executor import global_scope
        if scope is not None and not isinstance(scope, core._Scope):
            raise TypeError(
2350 2351
                "`scope` should be None or `paddle.static.Scope` type, but received {}."
                .format(type(scope)))
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363

        if scope is None:
            scope = global_scope()
        var_temp = scope.find_var(self.name)
        if var_temp is None:
            raise ValueError("Can not find Variable '{}' in the Scope.".format(
                self.name))
        t = var_temp.get_tensor()
        return t

    def set_value(self, value, scope=None):
        '''
2364
        Set the value to the tensor in given scope.
2365 2366 2367

        Args:
            value(Tensor/ndarray) : The value to be set.
2368
            scope(Scope, optional) : If `scope` is None, it will be set to global scope
2369 2370 2371 2372 2373
                obtained through 'paddle.static.global_scope()'. Otherwise, use `scope`.
                Default: None

        Returns:
            None
2374

2375 2376 2377 2378
        Examples:
            .. code-block:: python

                import paddle
2379
                import paddle.static as static
2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
                import numpy as np

                paddle.enable_static()

                x = static.data(name="x", shape=[10, 10], dtype='float32')

                y = static.nn.fc(x, 10, name='fc')
                place = paddle.CPUPlace()
                exe = static.Executor(place)
                prog = paddle.static.default_main_program()
                exe.run(static.default_startup_program())
                inputs = np.ones((10, 10), dtype='float32')
                exe.run(prog, feed={'x': inputs}, fetch_list=[y, ])
                path = 'temp/tensor_'
                for var in prog.list_vars():
                    if var.persistable:
                        t = var.get_value()
                        paddle.save(t, path+var.name+'.pdtensor')

                for var in prog.list_vars():
                    if var.persistable:
                        t_load = paddle.load(path+var.name+'.pdtensor')
                        var.set_value(t_load)
        '''

        # The 'framework' is a low-level module, and 'executor'
2406
        # can not be imported at the begainning of this file.
2407 2408 2409 2410 2411
        # Therefore, the above two modules are dynamically imported.
        from .executor import global_scope

        if not (isinstance(value, np.ndarray) or hasattr(value, '__array__')):
            raise TypeError(
2412 2413
                "`value` should be `numpy.ndarray` or `LoDTensor`, but received {}."
                .format(type(value)))
2414 2415 2416

        if scope is not None and not isinstance(scope, core._Scope):
            raise TypeError(
2417 2418
                "`scope` should be None or `paddle.static.Scope` type, but received {}."
                .format(type(scope)))
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448

        if scope is None:
            scope = global_scope()

        var_temp = scope.find_var(self.name)
        if var_temp is None:
            raise ValueError("Can not find Variable '{}' in the Scope.".format(
                self.name))

        t = var_temp.get_tensor()

        if hasattr(value, 'shape'):
            if isinstance(value.shape, (MethodType, FunctionType)):
                value_shape = value.shape()
            else:
                value_shape = value.shape
            if list(t.shape()) != list(value_shape):
                raise ValueError(
                    "{} expected a shape {}, but the received shape is {}.".
                    format(self.name, list(t.shape()), list(value_shape)))

        p = t._place()
        if p.is_cpu_place():
            place = core.CPUPlace()
        elif p.is_cuda_pinned_place():
            place = core.CUDAPinnedPlace()
        elif p.is_xpu_place():
            p = core.Place()
            p.set_place(t._place())
            place = core.XPUPlace(p.xpu_device_id())
2449 2450 2451 2452
        elif p.is_npu_place():
            p = core.Place()
            p.set_place(t._place())
            place = core.NPUPlace(p.npu_device_id())
2453 2454 2455 2456
        elif p.is_mlu_place():
            p = core.Place()
            p.set_place(t._place())
            place = core.MLUPlace(p.mlu_device_id())
2457 2458 2459 2460 2461 2462 2463
        else:
            p = core.Place()
            p.set_place(t._place())
            place = core.CUDAPlace(p.gpu_device_id())

        t.set(value, place)

2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488
    def size(self):
        """
        Returns the number of elements for current Variable, which is a int64 Variable with shape [1]

        Returns:
            Variable: the number of elements for current Variable

        Examples:
            .. code-block:: python

                import paddle

                paddle.enable_static()

                # create a static Variable
                x = paddle.static.data(name='x', shape=[3, 2, 1])

                # get the number of elements of the Variable
                y = x.size()
        """

        output = self.block.create_var(
            name=unique_name.generate_with_ignorable_key(self.name + "_size"),
            dtype=core.VarDesc.VarType.INT64)

2489 2490 2491
        self.block.append_op(type='size',
                             inputs={'Input': [self]},
                             outputs={'Out': [output]})
2492 2493
        return output

2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547
    def _set_attr(self, name, val):
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(int|str|list): the value of the attribute.
        """
        self._update_desc_attr(name, val)

    def _has_attr(self, name):
        """
        Whether this Variable has the attribute with the name `name` or not.

        Args:
            name(str): the attribute name.

        Returns:
            bool: True if has this attribute.
        """
        return self.desc.has_attr(name)

    def _remove_attr(self, name):
        self.desc.remove_attr(name)

    def _update_desc_attr(self, name, val):
        """
        Update the value of desc's attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(int|str|list): the value of the attribute.
        """
        self.desc._set_attr(name, val)

    @property
    def attr_names(self):
        """Get the names of all attributes defined."""
        return self.desc.attr_names()

    def _get_attr(self, name):
        """
        Get the attribute by name.

        Args:
            name(str): the attribute name.

        Returns:
            int|str|list: The attribute value. The return value
            can be any valid attribute type.
        """
        return self.desc.attr(name)

    @property
2548
    def dist_attr(self):
2549
        """
2550
        Get distributed attribute of this Variable.
2551
        """
2552
        return self.desc.dist_attr
2553

2554 2555
    @dist_attr.setter
    def dist_attr(self, dist_attr):
2556
        """
2557
        Set distributed attribute of this Variable.
2558
        """
2559
        self.desc.dist_attr = dist_attr
2560

Y
Yu Yang 已提交
2561

F
fengjiayi 已提交
2562 2563 2564
def get_all_op_protos():
    """
    Get all registered op proto from PaddlePaddle C++ end.
2565

2566 2567
    Returns:
       list: list of OpProto.
F
fengjiayi 已提交
2568 2569 2570 2571
    """
    protostrs = core.get_all_op_protos()
    ret_values = []
    for pbstr in protostrs:
2572
        op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr))
F
fengjiayi 已提交
2573 2574 2575 2576 2577
        ret_values.append(op_proto)
    return ret_values


class OpProtoHolder(object):
2578 2579 2580 2581
    """
    A global variable to hold all OpProtos from C++ as a map
    """

F
fengjiayi 已提交
2582 2583 2584 2585 2586 2587 2588 2589 2590
    @classmethod
    def instance(cls):
        if not hasattr(cls, '_instance'):
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        assert not hasattr(
            self.__class__,
2591
            '_instance'), 'Please use `instance()` to get OpProtoHolder object!'
F
fengjiayi 已提交
2592 2593 2594 2595 2596 2597
        op_protos = get_all_op_protos()
        self.op_proto_map = {}
        for proto in op_protos:
            self.op_proto_map[proto.type] = proto

    def get_op_proto(self, type):
2598 2599 2600 2601 2602 2603 2604 2605
        """
        Get OpProto by a type string.
        Args:
            type(str): The type that operator registered in C++ side.

        Returns(framework_pb2.OpProto): The OpProto

        """
Y
Yu Yang 已提交
2606 2607
        if type not in self.op_proto_map:
            raise ValueError("Operator \"%s\" has not been registered." % type)
F
fengjiayi 已提交
2608 2609
        return self.op_proto_map[type]

2610 2611
    def update_op_proto(self):
        op_protos = get_all_op_protos()
2612
        custom_op_names = []
2613 2614 2615
        for proto in op_protos:
            if proto.type not in self.op_proto_map:
                self.op_proto_map[proto.type] = proto
2616 2617 2618
                custom_op_names.append(proto.type)

        return custom_op_names
2619

2620 2621 2622 2623
    @staticmethod
    def generated_op_attr_names():
        return {
            core.op_proto_and_checker_maker.kOpRoleAttrName(),
S
sneaxiy 已提交
2624
            core.op_proto_and_checker_maker.kOpRoleVarAttrName(),
2625
            core.op_proto_and_checker_maker.kOpNameScopeAttrName(),
2626 2627
            core.op_proto_and_checker_maker.kOpCreationCallstackAttrName(),
            core.op_proto_and_checker_maker.kOpDeviceAttrName()
2628 2629
        }

F
fengjiayi 已提交
2630

X
Xin Pan 已提交
2631
class Operator(object):
2632
    """
2633 2634 2635 2636 2637 2638 2639
    In Fluid, all the operation are represented by Operator, and Operator
    is regarded as a build in an instruction of a Block. Users can use the
    build in instructions to describe their neural network.

    Args:
        block(Block): The block has the current operator.
        desc(core.OpDesc): The protobuf description of Operator.
C
chengduoZH 已提交
2640
        type(str): The type of operator. Default None.
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660
        inputs(dict): The input of this Operator. it is a dictionary, for every
            element, key is the input parameter name, and value is a list of
            variables. Default None.
        outputs(dict): The output of this Operator. it is a dictionary, for
            every element, key is the input parameter name, and value is a list
            of variables. Default None.
        attrs(dict): The attributes of this Operator. it is a dictionary, for
            every element, key is attribute name, and value is the attribute value.
            The attribute type should be as same as the type registered in C++ side.
            Default None.

    Returns:
        Operator: The initialized Operator.

    Raises:
        ValueError: If the passed input, output and attrs doesn't match the
            initializing Operator's that registered in C++ side.

    Notes:
        The constructor of operator should not be invoked directly. Use
W
Wu Yi 已提交
2661
        Block.append_op or Block._prepend_op instead.
2662 2663 2664 2665

    Examples:
        .. code-block:: python

2666
            import paddle.fluid as fluid
2667
            cur_program = fluid.Program()
2668 2669 2670 2671 2672
            cur_block = cur_program.current_block()
            # var1 += var2 + var3
            cur_block.append_op(type="sum",
                                inputs={"X": [var1, var2, var3]},
                                outputs={"Out": [var1]})
2673
    """
2674
    OP_WITHOUT_KERNEL_SET = {
2675 2676
        'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad',
        'conditional_block', 'while', 'send', 'recv', 'listen_and_serv',
2677
        'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify',
2678 2679
        'gen_bkcl_id', 'c_gen_bkcl_id', 'gen_nccl_id', 'c_gen_nccl_id',
        'c_comm_init', 'c_sync_calc_stream', 'c_sync_comm_stream',
W
WangXi 已提交
2680
        'queue_generator', 'dequeue', 'enqueue', 'heter_listen_and_serv',
B
Baibaifan 已提交
2681
        'c_wait_comm', 'c_wait_compute', 'c_gen_hccl_id', 'c_comm_init_hccl',
2682
        'copy_cross_scope', 'c_gen_cncl_id'
2683
    }
2684

Y
Yu Yang 已提交
2685 2686
    def __init__(self,
                 block,
Y
Yu Yang 已提交
2687
                 desc,
Y
Yu Yang 已提交
2688 2689 2690
                 type=None,
                 inputs=None,
                 outputs=None,
M
minqiyang 已提交
2691
                 attrs=None):
2692 2693 2694 2695 2696 2697 2698 2699 2700 2701
        # read attr type index from op proto to avoid unexpected type
        # conversions, e.g. narrowing conversion like double to float
        try:
            proto = OpProtoHolder.instance().get_op_proto(type)
            self._attr_types = {}
            for attr in proto.attrs:
                self._attr_types[attr.name] = attr.type
        except ValueError:
            pass

J
Jiabin Yang 已提交
2702
        if _non_static_mode():
2703 2704
            if type is None:
                raise ValueError(
2705
                    "`type` to initialized an Operator can not be None.")
J
Jiabin Yang 已提交
2706
            self._type = type
M
minqiyang 已提交
2707
            self.attrs = attrs if attrs else {}
2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
        else:
            self.block = block
            self.desc = desc
            # note: not add self.attrs here:
            # https://github.com/PaddlePaddle/Paddle/pull/12583#pullrequestreview-145093173
            op_attrs = attrs
            if op_attrs is None:
                op_attrs = dict()
            del attrs

2718 2719 2720
            # attr for static mode cuda graph
            self._cuda_graph_attr = _current_cuda_graph_mode

2721 2722 2723
            op_maker = core.op_proto_and_checker_maker

            if op_maker.kOpRoleAttrName() not in op_attrs:
2724 2725
                op_attrs[
                    op_maker.kOpRoleAttrName()] = self.block.program._op_role
2726 2727

            role_var_name = op_maker.kOpRoleVarAttrName()
2728 2729
            if len(self.block.program._op_role_var
                   ) != 0 and role_var_name not in op_attrs:
2730
                op_attrs[role_var_name] = self.block.program._op_role_var
2731 2732 2733 2734 2735

            if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
                del op_attrs[role_var_name]

            if len(self.desc.type()) != 0:
2736 2737 2738 2739 2740
                # NOTE(Aurelius84): prog.clone() will lead that var.op is always None,
                # we add this to fix the problem.
                for arg in self.desc.output_arg_names():
                    if block.has_var(arg) and block.var(arg).op is None:
                        block.var(arg).op = self
2741 2742 2743
                return
            if type is None:
                raise ValueError(
2744
                    "`type` to initialized an Operator can not be None.")
2745 2746
            else:
                callstack_var_name = op_maker.kOpCreationCallstackAttrName()
2747 2748 2749
                op_attrs[callstack_var_name] = []
                for frame in traceback.extract_stack():
                    op_attrs[callstack_var_name].append(
2750 2751 2752 2753
                        '  File "{}", line {}, in {}'.format(
                            frame[0], frame[1], frame[2]))
                    op_attrs[callstack_var_name].append('    {}'.format(
                        frame[3]))
2754 2755 2756 2757 2758 2759 2760

            self.desc.set_type(type)
            proto = OpProtoHolder.instance().get_op_proto(type)

            namescope_var_name = op_maker.kOpNameScopeAttrName()
            op_attrs[namescope_var_name] = _full_name_scope()

2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771
            # set device for op with kernels, give warning for op without kernels
            # when force_cpu and device_guard are used at the same time, a warning will be given.
            # TODO(zhangting2020): when force_cpu is removed, clear warning below.
            if _current_device is not None:
                if self._has_kernel(type):
                    op_device = op_maker.kOpDeviceAttrName()
                    op_attrs[op_device] = _current_device
                else:
                    warnings.warn("The Op(%s) is not support to set device." %
                                  type)
                if 'force_cpu' in op_attrs:
2772
                    if (type == 'less_than' and op_attrs['force_cpu'] != None
2773 2774 2775 2776 2777
                        ) or op_attrs['force_cpu'] != False:
                        warnings.warn(
                            "The Attr(force_cpu) of Op(%s) will be deprecated in the future, "
                            "please use 'device_guard' instead. 'device_guard' has higher priority when they are "
                            "used at the same time." % type)
2778 2779 2780 2781 2782
            if _current_pipeline_stage is not None:
                pipeline_attr_name = 'pipeline_stage' + core.kAutoParallelSuffix(
                )
                self._update_desc_attr(pipeline_attr_name,
                                       _current_pipeline_stage)
2783

2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796
            def find_name(var_list, name):
                for var_name in var_list:
                    if var_list[var_name] is not None and var_name == name:
                        return True
                return False

            if inputs is not None:
                for in_proto in proto.inputs:
                    found = find_name(inputs, in_proto.name)
                    assert found or in_proto.dispensable, "Input {} not found".format(
                        in_proto.name)
                    if found:
                        in_args = inputs[in_proto.name]
2797
                        if not isinstance(in_args, (list, tuple)):
2798 2799 2800 2801 2802 2803
                            in_args = [in_args]
                        if not in_proto.duplicable and len(in_args) > 1:
                            raise ValueError(
                                "Input %s expects only one input, but %d are given."
                                % (in_proto.name, len(in_args)))
                        in_arg_names = []
2804
                        for index, arg in enumerate(in_args):
2805 2806 2807 2808
                            if isinstance(arg, six.string_types):
                                in_arg_names.append(arg)
                            elif isinstance(arg, six.binary_type):
                                in_arg_names.append(arg.decode())
2809
                            elif isinstance(arg, (Variable, core.VarBase)):
2810
                                in_arg_names.append(cpt.to_text(arg.name))
2811
                            else:
2812 2813 2814 2815
                                raise TypeError(
                                    "The type of '%s' in operator %s should be "
                                    "one of [basestring(), str, Varibale] in python2, "
                                    "or one of [str, bytes, Variable] in python3."
2816 2817
                                    "but received : %s" %
                                    (in_proto.name, type, arg))
2818 2819 2820 2821 2822 2823 2824 2825 2826
                        self.desc.set_input(in_proto.name, in_arg_names)
                    else:
                        self.desc.set_input(in_proto.name, [])

            if outputs is not None:
                for m in proto.outputs:
                    if (m.name not in outputs) and m.dispensable:
                        continue
                    if not ((m.name in outputs) or m.dispensable):
2827 2828 2829 2830
                        raise ValueError(
                            ("Incorrect setting for output(s) of "
                             "operator \"%s\", should set: [%s].") %
                            (type, m.name))
2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842
                for out_proto in proto.outputs:
                    if out_proto.name not in outputs:
                        continue
                    out_args = outputs[out_proto.name]
                    if not isinstance(out_args, list):
                        out_args = [out_args]
                    if not out_proto.duplicable and len(out_args) > 1:
                        raise ValueError(
                            "Output %s expects only one output, but %d are given."
                            % (out_proto.name, len(out_args)))
                    out_arg_names = []
                    for arg in out_args:
2843 2844 2845 2846
                        if isinstance(arg, six.string_types):
                            out_arg_names.append(arg)
                        else:
                            out_arg_names.append(cpt.to_text(arg.name))
2847
                        # TODO(minqiyang): could we remove variable's op in static mode?
J
Jiabin Yang 已提交
2848
                        if not _non_static_mode():
2849 2850 2851 2852
                            if isinstance(arg, six.string_types):
                                block.var(arg).op = self
                            else:
                                arg.op = self
2853 2854
                    self.desc.set_output(out_proto.name, out_arg_names)

2855
            extra_attrs_map = core.get_op_extra_attrs(type)
2856 2857 2858 2859 2860
            if op_attrs is not None:
                if not isinstance(op_attrs, dict):
                    raise TypeError("'attrs' should be a dict.")
                for attr in proto.attrs:
                    attr_name = attr.name
2861 2862
                    if (attr_name
                            not in op_attrs) or (op_attrs[attr_name] is None):
2863 2864 2865
                        continue
                    attr_val = op_attrs[attr_name]
                    self._update_desc_attr(attr_name, attr_val)
2866 2867 2868 2869 2870 2871 2872
                for attr_name in extra_attrs_map.keys():
                    if (attr_name
                            not in op_attrs) or (op_attrs[attr_name] is None):
                        self._update_desc_attr(attr_name,
                                               extra_attrs_map[attr_name])
                    else:
                        self._update_desc_attr(attr_name, op_attrs[attr_name])
2873

J
jianghaicheng 已提交
2874 2875
            # proto.attrs doesn't include ipu_index
            if core.is_compiled_with_ipu():
2876
                if global_ipu_index >= 0:
J
jianghaicheng 已提交
2877 2878
                    self._update_desc_attr(ipu_index_attr_name,
                                           global_ipu_index)
2879
                if global_ipu_stage >= 0:
J
jianghaicheng 已提交
2880 2881 2882
                    self._update_desc_attr(ipu_stage_attr_name,
                                           global_ipu_stage)

2883 2884 2885 2886 2887
            self.desc.check_attrs()
            if self._has_kernel(type):
                self.desc.infer_var_type(self.block.desc)
                self.desc.infer_shape(self.block.desc)

W
Wu Yi 已提交
2888
    def _has_kernel(self, op_type):
2889 2890
        return op_type not in self.OP_WITHOUT_KERNEL_SET

Y
Yang Yang(Tony) 已提交
2891
    def to_string(self, throw_on_error):
2892
        """
2893 2894
        Get debug string.

2895
        Args:
2896 2897
            throw_on_error(bool): Whether to raise exception if self is not
                initialized.
2898

2899 2900
        Returns:
            str: The debug string.
2901 2902

        """
2903
        protostr = self.desc.serialize_to_string()
2904
        proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr))
Y
Yang Yang(Tony) 已提交
2905 2906
        return _debug_string_(proto, throw_on_error)

2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938
    def _to_readable_code(self, skip_op_callstack=True):
        """
        Get readable debug string of Operator.

        .. note::
            If you want to get the debug string in protobuf format,
            please use :code:`to_string` method.

        Args:
            skip_op_callstack(bool): whether to skip parsing Operator's attribute
                op_callstack, default value is True

        Returns:
            string: The formatted Operator string.

        Examples:
            .. code-block:: python

            import paddle.fluid as fluid

            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            var = cur_block.create_var(name="X",
                                       shape=[-1, 23, 48],
                                       dtype='float32')
            new_op = cur_block.append_op(type="abs",
                                inputs={"X": [var]},
                                outputs={"Out": [var]})
            print(new_op._to_readable_code())
        """
        assert isinstance(
            skip_op_callstack, bool
Z
zhangchunle 已提交
2939
        ), "skip_op_callstack parameter's type is error, expect bool, received {}".format(
2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
            type(skip_op_callstack))
        outputs_str = "{"
        for i in range(0, len(self.output_names)):
            outputs_str += "{name}=".format(name=self.output_names[i])
            o = self.output(self.output_names[i])
            outputs_str += "{value}".format(value=o)
            if i != len(self.output_names) - 1:
                outputs_str += ", "
        outputs_str += "}"

        inputs_str = "{"
        for i in range(0, len(self.input_names)):
            inputs_str += "{name}=".format(name=self.input_names[i])
            o = self.input(self.input_names[i])
            inputs_str += "{value}".format(value=o)

            if i != len(self.input_names) - 1:
                inputs_str += ", "
        inputs_str += "}"

        attr_names = sorted(self.attr_names)
        attrs_str = ""
        for i in range(0, len(attr_names)):
            name = attr_names[i]
            if skip_op_callstack and name == "op_callstack":
                continue

2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988
            attr_type = self.desc.attr_type(name, True)
            if attr_type == core.AttrType.VAR:
                attr_var_name = self.desc.attr(name, True).name()
                a = "{name} = Var['{value}']".format(name=name,
                                                     type=attr_type,
                                                     value=attr_var_name)
                attrs_str += a
                if i != len(attr_names) - 1:
                    attrs_str += ", "
                continue

            if attr_type == core.AttrType.VARS:
                attr_var_names = [
                    "'%s'" % var.name() for var in self.desc.attr(name, True)
                ]
                a = "{name} = Vars[{value}]".format(
                    name=name, type=attr_type, value=','.join(attr_var_names))
                attrs_str += a
                if i != len(attr_names) - 1:
                    attrs_str += ", "
                continue

2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006
            if attr_type == core.AttrType.BLOCK:
                a = "{name} = block[{value}]".format(
                    name=name, type=attr_type, value=self._block_attr_id(name))
                attrs_str += a
                if i != len(attr_names) - 1:
                    attrs_str += ", "
                continue

            if attr_type == core.AttrType.BLOCKS:
                a = "{name} = blocks{value}".format(
                    name=name,
                    type=attr_type,
                    value=self._blocks_attr_ids(name))
                attrs_str += a
                if i != len(attr_names) - 1:
                    attrs_str += ", "
                continue

3007
            # it is bytes of serialized protobuf
3008 3009 3010 3011
            if is_compiled_with_cinn(
            ) and self.type == 'cinn_launch' and name == 'compilation_key':
                key = self.desc.attr(name)
                v = core.get_serialize_comile_key(key)
3012 3013 3014 3015 3016 3017 3018 3019 3020
                prog = Program()
                prog = prog.parse_from_string(v)
                s = prog._to_readable_code()
                lines = s.split('\n')
                value = '\n'.join(['      ' + line for line in lines])
                value = '\n' + value
            else:
                value = self.desc.attr(name)

3021 3022 3023
            a = "{name} = {value}".format(name=name,
                                          type=attr_type,
                                          value=value)
3024

3025 3026 3027 3028
            attrs_str += a
            if i != len(attr_names) - 1:
                attrs_str += ", "

3029
        from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context
3030
        dist_context = get_default_distributed_context()
3031 3032
        dist_op = dist_context.get_dist_op_for_program(self)
        if dist_op is not None:
3033 3034
            attrs_str += ", {name} = {value}".format(name="dist_attr",
                                                     value=dist_op)
3035

3036 3037
        if outputs_str != "{}":
            op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".\
T
tangwei12 已提交
3038 3039
                format(outputs=outputs_str, op_type=self.type,
                       inputs=inputs_str, attrs=attrs_str)
3040 3041 3042 3043 3044
        else:
            op_str = "{op_type}(inputs={inputs}, {attrs})".\
                format(op_type=self.type, inputs=inputs_str, attrs=attrs_str)
        return op_str

Y
Yang Yang(Tony) 已提交
3045
    def __str__(self):
3046
        return self._to_readable_code()
3047 3048 3049

    __repr__ = __str__

F
fengjiayi 已提交
3050 3051
    @property
    def type(self):
3052
        return self.desc.type()
F
fengjiayi 已提交
3053 3054

    def input(self, name):
3055
        r"""
3056
        Get the input arguments according to the input parameter name.
3057

3058 3059
        Args:
            name(str): The input parameter name.
3060

3061 3062 3063
        Returns:
            list: return the list of argument names that associated with \
                the specific parameter name.
3064
        """
F
fengjiayi 已提交
3065 3066
        return self.desc.input(name)

W
Wu Yi 已提交
3067
    def _rename_input(self, old_name, new_name):
3068 3069 3070 3071 3072 3073 3074 3075 3076 3077
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's input.
            new_name(str): The new name of the Operator's input.

        Returns:
            None
        """
W
Wu Yi 已提交
3078
        self.desc._rename_input(old_name, new_name)
T
typhoonzero 已提交
3079

W
Wu Yi 已提交
3080
    def _rename_output(self, old_name, new_name):
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's output.
            new_name(str): The new name of the Operator's output.

        Returns:
            None
        """
W
Wu Yi 已提交
3091
        self.desc._rename_output(old_name, new_name)
T
typhoonzero 已提交
3092

F
fengjiayi 已提交
3093 3094 3095 3096
    @property
    def input_names(self):
        return self.desc.input_names()

T
typhoonzero 已提交
3097 3098 3099 3100 3101 3102 3103 3104
    @property
    def input_arg_names(self):
        return self.desc.input_arg_names()

    @property
    def output_arg_names(self):
        return self.desc.output_arg_names()

F
fengjiayi 已提交
3105
    def output(self, name):
3106
        r"""
3107
        Get output arguments by the output parameter name.
3108

3109 3110
        Args:
            name(str): The output parameter name.
3111

3112 3113 3114
        Returns:
            list: return the list of argument names associated with \
                the specific parameter name.
3115
        """
F
fengjiayi 已提交
3116 3117 3118 3119 3120 3121
        return self.desc.output(name)

    @property
    def output_names(self):
        return self.desc.output_names()

3122 3123 3124 3125 3126 3127 3128 3129
    @property
    def idx(self):
        for i, op in enumerate(self.block.ops):
            if op == self:
                return i
        raise ValueError(
            "Can't find op itself in it's block. It could be a bug of Paddle.")

F
fengjiayi 已提交
3130
    def has_attr(self, name):
3131
        """
3132 3133
        Whether this Operator has the attribute with name or not.

3134
        Args:
3135
            name(str): the attribute name.
3136

3137 3138
        Returns:
            bool: True if has this attribute.
3139 3140

        """
F
fengjiayi 已提交
3141 3142 3143
        return self.desc.has_attr(name)

    def attr_type(self, name):
3144
        """
3145
        Get the type of attribute by attribute's name.
3146

3147 3148
        Args:
            name(str): the attribute name.
3149

3150 3151
        Returns:
            core.AttrType: the attribute type.
3152
        """
3153
        return self.desc.attr_type(name, True)
F
fengjiayi 已提交
3154

W
Wu Yi 已提交
3155
    def _set_attr(self, name, val):
3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
G
gongweibao 已提交
3166 3167
        self._update_desc_attr(name, val)

3168 3169 3170
    def _remove_attr(self, name):
        self.desc.remove_attr(name)

G
gongweibao 已提交
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181
    def _update_desc_attr(self, name, val):
        """
        Update the value of desc's attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
3182 3183 3184 3185 3186
        if isinstance(val, Variable):
            self.desc.set_var_attr(name, val.desc)
        elif isinstance(val, list) and _all_is_type(val, Variable):
            self.desc.set_vars_attr(name, [v.desc for v in val])
        elif isinstance(val, Block):
Q
Qiyang Min 已提交
3187
            self.desc.set_block_attr(name, val.desc)
3188
        elif isinstance(val, list) and val and _all_is_type(val, Block):
3189
            self.desc.set_blocks_attr(name, [v.desc for v in val])
Q
Qiyang Min 已提交
3190 3191 3192 3193
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            self.desc.set_serialized_attr(name, val.serialize_to_string())
        else:
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
            self._update_desc_plain_attr(name, val)

    def _update_desc_plain_attr(self, name, val):
        desc = self.desc
        if not hasattr(self, "_attr_types") or (name not in self._attr_types):
            desc._set_attr(name, val)
            return

        type_index = self._attr_types[name]
        if type_index == core.AttrType.BOOL:
            desc._set_bool_attr(name, val)
        elif type_index == core.AttrType.INT:
            desc._set_int32_attr(name, val)
        elif type_index == core.AttrType.LONG:
            desc._set_int64_attr(name, val)
        elif type_index == core.AttrType.FLOAT:
            desc._set_float32_attr(name, val)
        # elif type_index == core.AttrType.FLOAT64:
        #     desc._set_float64_attr(name, val)
        elif type_index == core.AttrType.STRING:
            desc._set_str_attr(name, val)
        elif type_index == core.AttrType.BOOLS:
            desc._set_bools_attr(name, val)
        elif type_index == core.AttrType.INTS:
            desc._set_int32s_attr(name, val)
        elif type_index == core.AttrType.LONGS:
            desc._set_int64s_attr(name, val)
        elif type_index == core.AttrType.FLOATS:
            desc._set_float32s_attr(name, val)
        elif type_index == core.AttrType.FLOAT64S:
            desc._set_float64s_attr(name, val)
        elif type_index == core.AttrType.STRINGS:
            desc._set_strs_attr(name, val)
        else:
            # defaults to old methods
            desc._set_attr(name, val)
Y
yuyang18 已提交
3230

F
fengjiayi 已提交
3231 3232
    @property
    def attr_names(self):
3233
        return self.desc.attr_names(True)
F
fengjiayi 已提交
3234 3235

    def attr(self, name):
3236
        """
3237 3238
        Get the attribute by name.

3239
        Args:
3240
            name(str): the attribute name.
3241

3242 3243
        Returns:
            bool|int|str|float|list: The attribute value. The return value
3244 3245
            can be any valid attribute type.
        """
F
fengjiayi 已提交
3246
        return self.desc.attr(name)
Y
Yu Yang 已提交
3247

W
Wu Yi 已提交
3248
    def _block_attr_id(self, name):
3249
        """
G
gongweibao 已提交
3250
        Get the block attribute's id by name.
3251

3252 3253
        Args:
            name(str): the attribute name.
3254

3255 3256
        Returns:
            int: the block index.
3257
        """
W
Wu Yi 已提交
3258
        return self.desc._block_attr_id(name)
G
gongweibao 已提交
3259

W
Wu Yi 已提交
3260
    def _block_attr(self, name):
G
gongweibao 已提交
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270
        """
        Get the block attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            block: the block attribute.
        """

W
Wu Yi 已提交
3271
        id = self._block_attr_id(name)
G
gongweibao 已提交
3272 3273 3274
        assert (id >= 0 and id < len(self.block.program.blocks))
        return self.block.program.blocks[id]

W
Wu Yi 已提交
3275
    def _blocks_attr(self, name):
G
gongweibao 已提交
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285
        """
        Get the blocks attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks attribute.
        """
        attrs = []
W
Wu Yi 已提交
3286
        for i in self._blocks_attr_ids(name):
G
gongweibao 已提交
3287 3288 3289 3290 3291
            assert (i >= 0 and i < len(self.block.program.blocks))
            attrs.append(self.block.program.blocks[i])

        return attrs

W
Wu Yi 已提交
3292
    def _blocks_attr_ids(self, name):
G
gongweibao 已提交
3293 3294 3295 3296 3297 3298 3299 3300 3301 3302
        """
        Get the blocks attribute's ids by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks ids.
        """

W
Wu Yi 已提交
3303
        return self.desc._blocks_attr_ids(name)
Y
Yu Yang 已提交
3304

3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339
    def _var_attr(self, name):
        """
        Get the Variable attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            Variable: the Variable attribute.
        """
        attr_type = self.desc.attr_type(name, True)
        assert attr_type == core.AttrType.VAR, "Required type attr({}) is Variable, but received {}".format(
            name, attr_type)
        attr_var_name = self.desc.attr(name, True).name()
        return self.block._var_recursive(attr_var_name)

    def _vars_attr(self, name):
        """
        Get the Variables attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            Variables: the Variables attribute.
        """
        attr_type = self.desc.attr_type(name, True)
        assert attr_type == core.AttrType.VARS, "Required type attr({}) is list[Variable], but received {}".format(
            name, attr_type)
        attr_vars = [
            self.block._var_recursive(var.name())
            for var in self.desc.attr(name, True)
        ]
        return attr_vars

J
JiayiFeng 已提交
3340
    def all_attrs(self):
F
fengjiayi 已提交
3341
        """
3342 3343 3344
        Get the attribute dict.

        Returns:
G
gongweibao 已提交
3345
            dict: The Operator's attribute dict, name->attr.
F
fengjiayi 已提交
3346 3347 3348 3349
        """
        attr_names = self.attr_names
        attr_map = {}
        for n in attr_names:
3350
            attr_type = self.desc.attr_type(n, True)
G
gongweibao 已提交
3351
            if attr_type == core.AttrType.BLOCK:
W
Wu Yi 已提交
3352
                attr_map[n] = self._block_attr(n)
3353
            elif attr_type == core.AttrType.BLOCKS:
W
Wu Yi 已提交
3354
                attr_map[n] = self._blocks_attr(n)
3355 3356 3357 3358 3359 3360
            elif attr_type == core.AttrType.VAR:
                attr_map[n] = self._var_attr(n)
            elif attr_type == core.AttrType.VARS:
                attr_map[n] = self._vars_attr(n)
            else:
                attr_map[n] = self.attr(n)
G
gongweibao 已提交
3361

F
fengjiayi 已提交
3362 3363
        return attr_map

3364 3365 3366
    def _is_optimize_op(self):
        op_maker = core.op_proto_and_checker_maker
        OPTIMIZE = core.op_proto_and_checker_maker.OpRole.Optimize
3367 3368 3369 3370

        if not self.desc.has_attr(op_maker.kOpRoleAttrName()):
            return False

3371 3372 3373
        op_role = self.desc.attr(op_maker.kOpRoleAttrName())
        if op_role & int(OPTIMIZE):
            return True
3374 3375 3376 3377 3378 3379 3380 3381

        return False

    def _is_backward_op(self):
        op_maker = core.op_proto_and_checker_maker
        BACKWARD = core.op_proto_and_checker_maker.OpRole.Backward

        if not self.desc.has_attr(op_maker.kOpRoleAttrName()):
3382 3383
            return False

3384 3385 3386 3387 3388 3389
        op_role = self.desc.attr(op_maker.kOpRoleAttrName())
        if op_role & int(BACKWARD):
            return True

        return False

3390
    @property
3391
    def dist_attr(self):
3392
        """
3393
        Get distributed attribute of this Variable.
3394
        """
3395
        return self.desc.dist_attr
3396

3397 3398
    @dist_attr.setter
    def dist_attr(self, dist_attr):
3399
        """
3400
        Set distributed attribute of this Variable.
3401
        """
3402
        self.desc.dist_attr = dist_attr
3403

Y
Yu Yang 已提交
3404

Y
Yu Yang 已提交
3405
class Block(object):
3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419
    """
    In Fluid, a Program is consistence of multi-Block, and Block stores
    VarDesc and OpDesc. In a specific Block, a VarDesc have a unique name.
    One block could have some child blocks, and child block's name scopes
    should inherit the parent's so that OpDesc in child block can reference
    a VarDesc that is stored in the parent block.
    Please reference the framework.proto for details.

    Args:
        program(Program): The Program that the Block belongs to.
        idx(int): The block's id in the Program.

    Notes:
        The constructor of Block should not be invoked directly. Please
W
Wu Yi 已提交
3420
        use `Program._create_block()` to create a block.
3421 3422 3423 3424

    Examples:
        .. code-block:: python

3425 3426 3427
            import paddle.fluid as fluid

            cur_program = fluid.Program()
3428 3429 3430 3431 3432 3433 3434 3435 3436
            cur_block = cur_program.current_block()
            var = cur_block.create_var(name="X",
                                       shape=[-1, 23, 48],
                                       dtype='float32')
            cur_block.append_op(type="abs",
                                inputs={"X": [var]},
                                outputs={"Out": [var]})
    """

Y
Yu Yang 已提交
3437
    def __init__(self, program, idx):
Y
Yu Yang 已提交
3438
        self.desc = program.desc.block(idx)
3439
        self.vars = collections.OrderedDict()  # var_name --> var
Q
qiaolongfei 已提交
3440
        self.ops = list()  # operator list
Y
Yu Yang 已提交
3441
        self.program = program
3442
        self.removed_vars = collections.OrderedDict()
Y
Yu Yang 已提交
3443

3444
    def __str__(self):
3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
        return self._to_readable_code()

    def _to_readable_code(self, skip_op_callstack=True):
        """
        Get readable debug string of Block.

        .. note::
            If you want to get the debug string in protobuf format,
            please use :code:`to_string` method.

        Args:
            skip_op_callstack(bool): whether to skip parsing Operator's attribute
                op_callstack, default value is True

        Returns:
            string: The formatted Block string.

        Examples:
            .. code-block:: python

            import paddle.fluid as fluid

            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_var = cur_block.create_var(name="X",
                                           shape=[-1, 23, 48],
                                           dtype='float32')
            new_op = cur_block.append_op(type="abs",
                                inputs={"X": [new_var]},
                                outputs={"Out": [new_var]})
            print(cur_block._to_readable_code())
        """
        assert isinstance(
            skip_op_callstack, bool
Z
zhangchunle 已提交
3479
        ), "skip_op_callstack parameter's type is error, expect bool, received {}".format(
3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490
            type(skip_op_callstack))
        block_str = "{ // block "
        block_str += "{}\n".format(self.idx)
        for var in list(self.vars.values()):
            block_str += "    {}\n".format(var._to_readable_code())
        block_str += "\n"
        for op in self.ops:
            block_str += "    {}\n".format(
                op._to_readable_code(skip_op_callstack))
        block_str += "}"
        return block_str
Y
Yang Yang(Tony) 已提交
3491

F
fengjiayi 已提交
3492 3493
    def to_string(self, throw_on_error, with_details=False):
        """
3494 3495
        Get debug string.

F
fengjiayi 已提交
3496 3497
        Args:
            throw_on_error(bool): raise exception when self is not initialized
3498
                when throw_on_error is True.
F
update  
fengjiayi 已提交
3499
            with_details(bool): more details about variables and parameters
3500 3501
                (e.g. trainable, optimize_attr, ...) will be printed when
                with_details is True. Default False.
F
fengjiayi 已提交
3502

3503 3504
        Returns:
            str: The debug string.
F
fengjiayi 已提交
3505
        """
3506 3507
        assert isinstance(throw_on_error, bool) and isinstance(
            with_details, bool)
F
fengjiayi 已提交
3508
        if with_details:
F
fengjiayi 已提交
3509
            re_add_indent = re.compile(r"\n(.)")
F
fengjiayi 已提交
3510 3511
            res_str = "blocks {\n  idx: %d\n  parent_idx: %d" % (
                self.idx, self.parent_idx)
3512
            for var in list(self.vars.values()):
F
fengjiayi 已提交
3513
                res_str += "\n  vars {\n    %s  }" % re_add_indent.sub(
F
update  
fengjiayi 已提交
3514
                    r"\n    \1", var.to_string(throw_on_error, with_details))
F
fengjiayi 已提交
3515
            for op in self.ops:
F
fengjiayi 已提交
3516 3517
                res_str += "\n  ops {\n    %s  }" % re_add_indent.sub(
                    r"\n    \1", op.to_string(throw_on_error))
F
fengjiayi 已提交
3518 3519 3520
            res_str += "\n}"
        else:
            protostr = self.desc.serialize_to_string()
3521 3522
            proto = framework_pb2.BlockDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
3523 3524
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
3525 3526 3527

    __repr__ = __str__

Y
Yu Yang 已提交
3528 3529
    @property
    def parent_idx(self):
Y
Yu Yang 已提交
3530
        return self.desc.parent
Y
Yu Yang 已提交
3531

Y
Yu Yang 已提交
3532 3533 3534 3535
    @property
    def forward_block_idx(self):
        return self.desc.get_forward_block_idx()

W
Wu Yi 已提交
3536
    def _set_forward_block_idx(self, idx):
3537 3538 3539 3540 3541 3542 3543 3544 3545
        """
        Set the forward block Idx.

        Args:
            idx(int): the block index.

        Returns:
            None
        """
W
Wu Yi 已提交
3546
        self.desc._set_forward_block_idx(idx)
Y
Yu Yang 已提交
3547

3548 3549 3550 3551 3552 3553 3554 3555
    @property
    def backward_block_idx(self):
        cur_block_idx = self.idx
        for block in self.program.blocks:
            if block.forward_block_idx == cur_block_idx:
                return block.idx
        return -1

Y
Yu Yang 已提交
3556 3557
    @property
    def idx(self):
Y
Yu Yang 已提交
3558
        return self.desc.id
Y
Yu Yang 已提交
3559

Q
Qiao Longfei 已提交
3560
    def var(self, name):
3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573
        """
        Get a Variable by name from this block.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: The If input's type is not str, or this block
                doesn't have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
3574
        if not isinstance(name, six.string_types):
M
minqiyang 已提交
3575 3576 3577
            raise TypeError(
                "var require string as parameter, but get %s instead." %
                (type(name)))
Y
Yu Yang 已提交
3578 3579
        v = self.vars.get(name, None)
        if v is None:
Q
Qiao Longfei 已提交
3580
            raise ValueError("var %s not in this block" % name)
Y
Yu Yang 已提交
3581
        return v
Q
Qiao Longfei 已提交
3582

X
Xin Pan 已提交
3583
    def _find_var_recursive(self, name):
3584 3585 3586 3587 3588 3589 3590
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Returns:
X
Xin Pan 已提交
3591
            Variable: the Variable with the giving name. Or None if not found.
3592
        """
Y
Yu Yang 已提交
3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616
        frontier = list()
        visited = set()

        frontier.append(self)

        prog = self.program

        while len(frontier) != 0:  # BFS
            cur = frontier[0]
            frontier = frontier[1:]

            if id(cur) in visited:
                continue

            if cur.has_var(name):
                return cur.var(name)

            if cur.parent_idx != -1:
                frontier.append(prog.block(cur.parent_idx))

            if cur.forward_block_idx != -1:
                frontier.append(prog.block(cur.forward_block_idx))

            visited.add(id(cur))
X
Xin Pan 已提交
3617
        return None
Y
Yu Yang 已提交
3618

X
Xin Pan 已提交
3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637
    def _var_recursive(self, name):
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: this block and this parent block doesn't
                have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
        var = self._find_var_recursive(name)
        if var:
            return var
        else:
            raise ValueError("Var {0} is not found recursively".format(name))
F
fengjiayi 已提交
3638

Q
Qiao Longfei 已提交
3639
    def all_parameters(self):
3640
        return list(self.iter_parameters())
3641

3642
    def iter_parameters(self):
M
minqiyang 已提交
3643
        return (item[1] for item in six.iteritems(self.vars)
3644
                if isinstance(item[1], Parameter))
Q
Qiao Longfei 已提交
3645

Y
Yu Yang 已提交
3646
    def create_var(self, *args, **kwargs):
J
Jiabin Yang 已提交
3647
        if _non_static_mode():
L
Leo Chen 已提交
3648 3649
            var = _varbase_creator(*args, **kwargs)
        else:
3650 3651 3652
            var = Variable(block=self, *args, **kwargs)
            if 'initializer' in kwargs:
                kwargs['initializer'](var, self)
Q
Qiao Longfei 已提交
3653
        return var
Y
Yu Yang 已提交
3654

Q
Qiao Longfei 已提交
3655 3656 3657
    def has_var(self, name):
        return name in self.vars

W
Wu Yi 已提交
3658
    def _rename_var(self, name, new_name):
T
typhoonzero 已提交
3659 3660
        """
        Rename variable in vars and ops' inputs and outputs
3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672

        Args:
            name(str): the name that need to be renamed.
            new_name(str): the name that need to rename to.

        Raises:
            ValueError: If this block doesn't have this the giving name,
                or the type of the var with the giving name is not Parameter
                or Variable.

        Returns:
            Variable: the Variable with the giving name.
T
typhoonzero 已提交
3673
        """
M
minqiyang 已提交
3674 3675
        name = cpt.to_text(name)
        new_name = cpt.to_text(new_name)
M
minqiyang 已提交
3676

T
typhoonzero 已提交
3677
        if not self.has_var(name):
3678
            raise ValueError("var %s is not in current block" % name)
T
wip  
typhoonzero 已提交
3679 3680
        v = self.var(name)
        if type(v) == Parameter:
T
typhoonzero 已提交
3681
            var_type = "Parameter"
T
wip  
typhoonzero 已提交
3682 3683 3684 3685 3686 3687
            stop_gradient = v.stop_gradient
            trainable = v.trainable
            optimize_attr = v.optimize_attr
            regularizer = v.regularizer
            error_clip = v.error_clip
        elif type(v) == Variable:
T
typhoonzero 已提交
3688
            var_type = "Variable"
T
wip  
typhoonzero 已提交
3689 3690 3691 3692
            error_clip = v.error_clip
            stop_gradient = v.stop_gradient
        else:
            raise ValueError("unsupported var type: %s", type(v))
T
typhoonzero 已提交
3693
        orig_var_type = v.type
3694
        self.desc._rename_var(name.encode(), new_name.encode())
W
Wu Yi 已提交
3695
        # NOTE: v is destroyed by C++ after calling _rename_var.
3696
        d = self.desc.find_var(new_name.encode())
T
typhoonzero 已提交
3697
        if var_type == "Parameter":
L
Leo Chen 已提交
3698
            if in_dygraph_mode():
3699 3700 3701 3702 3703 3704 3705 3706 3707
                var = EagerParamBase(d.shape(),
                                     d.dtype(),
                                     type=orig_var_type,
                                     name=new_name,
                                     stop_gradient=stop_gradient,
                                     trainable=trainable,
                                     optimize_attr=optimize_attr,
                                     regularizer=regularizer,
                                     error_clip=error_clip)
3708
            else:
J
Jiabin Yang 已提交
3709
                if _in_legacy_dygraph():
3710 3711 3712 3713 3714 3715 3716 3717 3718
                    var = ParamBase(d.shape(),
                                    d.dtype(),
                                    type=orig_var_type,
                                    name=new_name,
                                    stop_gradient=stop_gradient,
                                    trainable=trainable,
                                    optimize_attr=optimize_attr,
                                    regularizer=regularizer,
                                    error_clip=error_clip)
J
Jiabin Yang 已提交
3719
                else:
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729
                    var = Parameter(self,
                                    d.shape(),
                                    d.dtype(),
                                    type=orig_var_type,
                                    name=new_name,
                                    stop_gradient=stop_gradient,
                                    trainable=trainable,
                                    optimize_attr=optimize_attr,
                                    regularizer=regularizer,
                                    error_clip=error_clip)
T
typhoonzero 已提交
3730
        elif var_type == "Variable":
3731 3732 3733 3734 3735
            var = Variable(self,
                           type=orig_var_type,
                           name=new_name,
                           error_clip=error_clip,
                           stop_gradient=stop_gradient)
T
wip  
typhoonzero 已提交
3736

W
Wu Yi 已提交
3737
        # rename the python side, _sync_with_cpp will only add
T
wip  
typhoonzero 已提交
3738 3739 3740
        # new vars/ops to python side.
        self.vars[new_name] = var
        del self.vars[name]
W
Wu Yi 已提交
3741
        self._sync_with_cpp()
3742
        return var
T
typhoonzero 已提交
3743

3744 3745 3746
    def _remove_var(self, name, sync=True):
        if sync == True:
            self._sync_with_cpp()
3747
        self.desc._remove_var(name.encode())
3748 3749
        del self.vars[name]

Y
Yu Yang 已提交
3750 3751
    def create_parameter(self, *args, **kwargs):
        global_block = self.program.global_block()
3752
        param = None
L
Leo Chen 已提交
3753
        if in_dygraph_mode():
J
Jiabin Yang 已提交
3754
            param = EagerParamBase(*args, **kwargs)
L
Leo Chen 已提交
3755
        else:
J
Jiabin Yang 已提交
3756 3757 3758 3759
            if _in_legacy_dygraph():
                param = ParamBase(*args, **kwargs)
            else:
                param = Parameter(global_block, *args, **kwargs)
3760

3761
        if 'initializer' in kwargs:
3762 3763 3764 3765 3766

            def _is_inited_by(block, var):
                init_ops = []
                for op in block.ops:
                    if var.name in op.output_arg_names:
3767
                        # In startup_program, "c_broadcast" and "c_sync_comm_stream"
T
tangwei12 已提交
3768
                        # are treated as initialization ops that cause error.
3769
                        # Think of "c_broadcast" and "c_sync_comm_stream" as a special case here.
3770 3771 3772 3773 3774
                        # NOTE: "coalesce_tensor" is a special case for rnn with cudnn support
                        if op.type in [
                                "c_broadcast", "c_sync_comm_stream",
                                "coalesce_tensor"
                        ]:
3775
                            continue
3776 3777 3778 3779 3780 3781 3782 3783
                        init_ops.append(op)
                return init_ops

            initializer = kwargs['initializer']
            init_ops = _is_inited_by(global_block, param)
            init_ops_len = len(init_ops)
            if init_ops_len > 1:
                raise RuntimeError("param " + param.name +
3784 3785
                                   " is inited by multiple init ops " +
                                   str(init_ops))
3786
            elif init_ops_len == 1:
3787
                # TODO already inited, do nothing, should log a warning
3788 3789 3790
                pass
            else:
                initializer(param, self)
Q
Qiao Longfei 已提交
3791
        return param
Y
Yu Yang 已提交
3792

Y
Yu Yang 已提交
3793
    def append_op(self, *args, **kwargs):
3794 3795 3796 3797 3798 3799
        """
        Appends a new Operator according to the giving arguments.

        Returns:
            Operator: the append Operator.
        """
J
Jiabin Yang 已提交
3800
        if _non_static_mode():
3801
            attrs = kwargs.get("attrs", {})
Z
zyfncg 已提交
3802
            inplace_map = kwargs.get("inplace_map", None)
J
Jiabin Yang 已提交
3803
            type = kwargs.get("type", None)
3804 3805 3806 3807
            warnings.warn(
                "Op `%s` is executed through `append_op` under the dynamic mode, "
                "the corresponding API implementation needs to be upgraded to "
                "using `_C_ops` method." % type, DeprecationWarning)
3808 3809 3810 3811 3812 3813
            op = Operator(block=self,
                          desc=None,
                          type=type,
                          inputs=None,
                          outputs=None,
                          attrs=attrs)
3814

M
minqiyang 已提交
3815 3816 3817
            # record ops in tracer rather than blocks
            #
            # TODO(minqiyang): add op stop_gradient support in static mode too.
L
lujun 已提交
3818
            # currently, we only support stop_gradient in dygraph mode.
J
Jiabin Yang 已提交
3819

3820 3821 3822
            _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}),
                                       kwargs.get("outputs",
                                                  {}), attrs if attrs else {},
Z
zyfncg 已提交
3823 3824
                                       kwargs.get("stop_gradient", False),
                                       inplace_map)
M
minqiyang 已提交
3825
        else:
3826 3827
            from paddle.fluid.dygraph.base import param_guard

3828
            op_desc = self.desc.append_op()
3829 3830 3831 3832 3833 3834
            # NOTE(Aurelius84): In case of @to_static, all VarBase(s) should
            # be converted into Variable(s) with same name and block location.
            # This is ONE and ONLY logic of type transformation of dy2static.
            inputs = kwargs.get("inputs", None)
            outputs = kwargs.get("outputs", None)
            with param_guard(inputs), param_guard(outputs):
3835 3836 3837 3838 3839 3840
                op = Operator(block=self,
                              desc=op_desc,
                              type=kwargs.get("type", None),
                              inputs=inputs,
                              outputs=outputs,
                              attrs=kwargs.get("attrs", None))
3841

M
minqiyang 已提交
3842
            self.ops.append(op)
M
minqiyang 已提交
3843

3844 3845
        return op

W
Wu Yi 已提交
3846
    def _insert_op(self, index, *args, **kwargs):
3847 3848 3849 3850 3851 3852 3853 3854 3855
        """
        Insert a Operator according to the giving arguments.

        Args:
            index(int): the place that the operator to insert.

        Returns:
            Operator: the insert Operator.
        """
W
Wu Yi 已提交
3856
        self._sync_with_cpp()
F
fangshuixun007 已提交
3857
        return self._insert_op_without_sync(index, *args, **kwargs)
Q
qiaolongfei 已提交
3858

3859 3860
    def _insert_op_without_sync(self, index, *args, **kwargs):
        """
3861
        Insert an Operator according to the giving arguments,
3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875
        without sync_with_cpp to meke the compilation faster.

        Args:
            index(int): the place that the operator to insert.

        Returns:
            Operator: the insert Operator.
        """
        op_desc = self.desc._insert_op(index)
        op = Operator(block=self, desc=op_desc, *args, **kwargs)
        self.ops.insert(index, op)
        return op

    def _remove_op(self, index, sync=True):
3876 3877 3878 3879 3880 3881 3882 3883 3884
        """
        Remove the specific position operator.

        Args:
            index(int): the position that the operator to insert.

        Returns:
            None
        """
3885 3886
        if sync == True:
            self._sync_with_cpp()
W
Wu Yi 已提交
3887
        self.desc._remove_op(index, index + 1)
3888 3889
        del self.ops[index]

W
Wu Yi 已提交
3890
    def _slice_ops(self, start, end):
3891 3892 3893 3894 3895 3896 3897 3898 3899 3900
        """
        Return the Operator between start and end.

        Args:
            start(int): the start position.
            end(int): the end position.

        Returns:
            list: the Operators between start and end.
        """
Q
qiaolongfei 已提交
3901
        return self.ops[start:end]
Y
Yancey1989 已提交
3902

W
Wu Yi 已提交
3903
    def _prepend_op(self, *args, **kwargs):
J
Jiabin Yang 已提交
3904
        if _non_static_mode():
J
Jiabin Yang 已提交
3905 3906
            type = kwargs.get("type", None)
            attrs = kwargs.get("attrs", {})
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916
            op = Operator(self,
                          None,
                          type=type,
                          inputs=None,
                          outputs=None,
                          attrs=attrs)

            _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}),
                                       kwargs.get("outputs", {}),
                                       attrs if attrs else {},
M
minqiyang 已提交
3917
                                       kwargs.get("stop_gradient", False))
M
minqiyang 已提交
3918
        else:
3919
            op_desc = self.desc._prepend_op()
3920 3921 3922 3923 3924 3925
            op = Operator(self,
                          op_desc,
                          type=kwargs.get("type", None),
                          inputs=kwargs.get("inputs", None),
                          outputs=kwargs.get("outputs", None),
                          attrs=kwargs.get("attrs", None))
M
minqiyang 已提交
3926
            self.ops.insert(0, op)
3927

Y
Yu Yang 已提交
3928 3929
        return op

W
Wu Yi 已提交
3930
    def _sync_with_cpp(self):
3931
        """
3932 3933
        Sync from the desc on the c++ end. This method is used to synchronize
        the c++ desc instance generated by backward.
3934
        """
Q
Qiao Longfei 已提交
3935 3936 3937
        # sync variables from cpp
        for var in self.desc.all_vars():
            if not self.has_var(var.name()):
3938 3939 3940 3941
                is_stop_gradient = False
                if var.has_stop_gradient():
                    is_stop_gradient = var.stop_gradient()
                if var.has_is_parameter() and var.is_parameter():
3942 3943 3944 3945 3946 3947
                    self.create_parameter(name=var.name(),
                                          desc=var,
                                          type=var.type(),
                                          shape=var.shape(),
                                          dtype=var.dtype(),
                                          stop_gradient=is_stop_gradient)
3948
                else:
3949 3950 3951 3952
                    self.create_var(name=var.name(),
                                    desc=var,
                                    type=var.type(),
                                    stop_gradient=is_stop_gradient)
Q
Qiao Longfei 已提交
3953

3954
        # sync variables removed from c++ end
3955
        for var in list(self.vars.keys()):
3956
            if not self.desc.find_var(var.encode()):
3957 3958
                self.vars.pop(var)

Q
Qiao Longfei 已提交
3959
        # sync operators from cpp
3960 3961 3962 3963
        ops_in_cpp = []
        for op_idx in range(0, self.desc.op_size()):
            ops_in_cpp.append(self.desc.op(op_idx))

Y
Yu Yang 已提交
3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979
        if len(self.ops) != 0:
            first_op_in_python = self.ops[0].desc
            last_op_in_python = self.ops[len(self.ops) - 1].desc
            start_index = None
            end_index = None
            for index in range(len(ops_in_cpp)):
                if first_op_in_python == ops_in_cpp[index]:
                    start_index = index
                if last_op_in_python == ops_in_cpp[index]:
                    end_index = index
            assert start_index is not None
            assert end_index is not None
            assert start_index <= end_index
        else:
            start_index = 0
            end_index = -1
Q
Qiao Longfei 已提交
3980 3981 3982 3983 3984

        # sync ops append to the head of cpp_ops
        for index in range((start_index - 1 - 1), -1, -1):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
Q
qiaolongfei 已提交
3985
            self.ops.insert(0, op)
Q
Qiao Longfei 已提交
3986 3987 3988 3989 3990 3991 3992

        # sync ops append to the end of cpp_ops
        for index in range((end_index + 1), len(ops_in_cpp)):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.append(op)

3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005
        # sync ops removed from c++ end
        if end_index != -1 and end_index < len(self.ops):
            ops_in_cpp_index = 0
            ops_in_python_index = 0
            while ops_in_python_index < len(
                    self.ops) and ops_in_cpp_index < len(ops_in_cpp):
                if self.ops[ops_in_python_index].desc != ops_in_cpp[
                        ops_in_cpp_index]:
                    del self.ops[ops_in_python_index]
                else:
                    ops_in_cpp_index += 1
                    ops_in_python_index += 1

Q
Qiao Longfei 已提交
4006 4007 4008 4009
        assert len(self.ops) == len(ops_in_cpp)
        for index in range(len(self.ops)):
            assert self.ops[index].desc == ops_in_cpp[index]

W
Wu Yi 已提交
4010
    def _copy_param_info_from(self, other):
4011
        """
4012 4013
        Copy the information of parameters from the other block.

4014
        Args:
4015 4016 4017 4018 4019
            other(Block): the other block.

        Raises:
            ValueError: If type of input is not Block, or the `other` and this
                block is not in the same topology.
4020 4021 4022 4023 4024

        Returns:
            None
        """
        if not isinstance(other, Block):
W
Wu Yi 已提交
4025 4026
            raise TypeError(
                "_copy_param_info_from should be invoked with Block")
4027
        for p in other.iter_parameters():
4028 4029 4030
            assert isinstance(p, Parameter)
            v = self.vars.get(p.name, None)
            if v is None:
4031 4032
                # if the Parameter is pruned, v may be None
                continue
4033
            assert isinstance(v, Variable)
4034
            new_p = None
L
Leo Chen 已提交
4035
            if in_dygraph_mode():
4036 4037 4038 4039 4040 4041 4042 4043 4044 4045
                new_p = EagerParamBase(shape=v.shape,
                                       dtype=v.dtype,
                                       type=v.type,
                                       lod_level=v.lod_level,
                                       stop_gradient=p.stop_gradient,
                                       trainable=p.trainable,
                                       optimize_attr=p.optimize_attr,
                                       regularizer=p.regularizer,
                                       error_clip=p.error_clip,
                                       name=v.name)
4046
            else:
J
Jiabin Yang 已提交
4047
                if _in_legacy_dygraph():
4048 4049 4050 4051 4052 4053 4054 4055 4056 4057
                    new_p = ParamBase(shape=v.shape,
                                      dtype=v.dtype,
                                      type=v.type,
                                      lod_level=v.lod_level,
                                      stop_gradient=p.stop_gradient,
                                      trainable=p.trainable,
                                      optimize_attr=p.optimize_attr,
                                      regularizer=p.regularizer,
                                      error_clip=p.error_clip,
                                      name=v.name)
J
Jiabin Yang 已提交
4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071
                else:
                    new_p = Parameter(
                        block=self,
                        shape=v.shape,
                        dtype=v.dtype,
                        type=v.type,
                        lod_level=v.lod_level
                        if v.type == core.VarDesc.VarType.LOD_TENSOR else None,
                        stop_gradient=p.stop_gradient,
                        trainable=p.trainable,
                        optimize_attr=p.optimize_attr,
                        regularizer=p.regularizer,
                        error_clip=p.error_clip,
                        name=v.name)
4072 4073
            self.vars[new_p.name] = new_p

4074
    def _clone_variable(self, var, force_persistable=True):
4075 4076
        """
        Clone a variable into current block.
4077

4078 4079
        Args:
            var: the variable to be cloned.
4080 4081 4082
            force_persistable(bool): True means setting the result variable to being persistable.
                                     False means setting the persistable the same with that of input var.
                                     default: True.
4083 4084

        Returns:
4085
            Variable: the new  variable cloned from 'var' in current block.
4086 4087
        """
        assert isinstance(var, Variable)
T
update  
typhoonzero 已提交
4088 4089 4090
        ret_var = None
        # make STEP_SCOPES var can be safely cloned.
        if var.type == core.VarDesc.VarType.STEP_SCOPES:
4091 4092 4093
            ret_var = self.create_var(name=var.name,
                                      persistable=var.persistable,
                                      type=var.type)
T
tangwei12 已提交
4094
        elif var.type == core.VarDesc.VarType.RAW:
4095 4096 4097
            ret_var = self.create_var(name=var.name,
                                      persistable=var.persistable,
                                      type=var.type)
T
typhoonzero 已提交
4098 4099 4100 4101 4102 4103
        elif var.type == core.VarDesc.VarType.SELECTED_ROWS:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
4104
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
4105 4106
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
4107 4108 4109 4110 4111 4112 4113
        else:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
                lod_level=var.lod_level,
4114
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
4115 4116
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
4117
        return ret_var
4118

Y
Yu Yang 已提交
4119

4120 4121 4122 4123
# NOTE(zjl): you should be careful that after you call this method,
# some Python Variable and all Python Operators should not be used
# again. Because all Python Variables and all Python Operators are
# re-constructed inside this method. The underlying VarDesc(OpDesc)
4124
# of some old Python Variables(all old Python Operators) may have
4125
# been destructed.
4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141
def _apply_pass(main_program,
                startup_program,
                pass_name,
                pass_attrs={},
                pass_attr_types={}):
    assert isinstance(pass_attrs, dict), "pass_attrs must be dict"
    assert isinstance(pass_attr_types, dict), "pass_attr_types must be dict"
    tmp_main_program = core.ProgramDesc(main_program.desc)
    tmp_startup_program = core.ProgramDesc(startup_program.desc)
    attrs = core.apply_pass(tmp_main_program, tmp_startup_program, pass_name,
                            pass_attrs, pass_attr_types)
    main_program._rebuild_from_desc(tmp_main_program)
    startup_program._rebuild_from_desc(tmp_startup_program)
    return attrs


4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
class IrNode(object):
    """
    Python IrNode. Beneath it is a core.Node, which is used for Ir Pass.
    """

    def __init__(self, node):
        """
        Construct an IrNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node,
                          core.Node), 'node must be the instance of core.Node.'
        self.node = node

    def name(self):
        """
        Return the node name.

        Returns:
            str: node name.
        """
        return self.node.name()

    def node_type(self):
        """
        Return the node type.

        Returns:
            core.Node.Type: node type(core.Node.Type.Operation or core.Node.Type.Variable).
        """
        return self.node.node_type()

    def var(self):
        """
        Return the node variable description.

        Returns:
            core.VarDesc: node variable description.
        """
        return self.node.var()

    def op(self):
        """
        Return the node operator description.

        Returns:
            core.OpDesc: node operator description.
        """
        return self.node.op()

    def id(self):
        """
        Return the node id.

        Returns:
            int: node id.
        """
        return self.node.id()

    def is_op(self):
        """
        If the node is an operator, then return true.

        Returns:
            bool: indicate whether the node is an operator.
        """
        return self.node.is_op()

    def is_var(self):
        """
        If the node is a variable, then return true.

        Returns:
            bool: indicate whether the node is a variable.
        """
        return self.node.is_var()

    def is_ctrl_var(self):
        """
        If the node is a control dependence variable, then return true.

        Returns:
            bool: indicate whether the node is a control dependence variable.
        """
        return self.node.is_ctrl_var()

    def clear_inputs(self):
        """
        Clear the node inputs. After executing the `clear_inputs` function,
        the node inputs will be empty.
        """
        self.node.clear_inputs()

4237
    def remove_input_by_id(self, node_id):
4238 4239 4240 4241 4242 4243
        """
        Remove a node from inputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
4244
        self.node.remove_input(node_id)
4245

4246
    def remove_input(self, node):
4247 4248 4249 4250
        """
        Remove a node from inputs.

        Args:
4251
            node(IrNode): the node being removed.
4252
        """
4253
        self.node.remove_input(node.node)
4254

4255
    def append_input(self, node):
4256 4257 4258 4259
        """
        Append a node in inputs.

        Args:
4260
            node(IrNode): the node being appended.
4261
        """
4262
        self.node.append_input(node.node)
4263 4264 4265 4266 4267 4268 4269 4270

    def clear_outputs(self):
        """
        Clear the node outputs. After executing the `clear_outputs` function,
        the node outputs will be empty.
        """
        self.node.clear_outputs()

4271
    def remove_output_by_id(self, node_id):
4272 4273 4274 4275 4276 4277
        """
        Remove a node from outputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
4278
        self.node.remove_output(node_id)
4279

4280
    def remove_output(self, node):
4281 4282 4283 4284
        """
        Remove a node from outputs.

        Args:
4285
            node(IrNode): the node being removed.
4286
        """
4287
        self.node.remove_output(node.node)
4288

4289
    def append_output(self, node):
4290 4291 4292 4293
        """
        Append a node in outputs.

        Args:
4294
            node(IrNode): the node being appended.
4295
        """
4296
        self.node.append_output(node.node)
4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343

    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrNode): node inputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrNode): node outputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.outputs]


class IrVarNode(IrNode):
    """
    Python IrVarNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrVarNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_var(), \
            'node must be the instance of core.Node and it must be a variable node.'
        super(IrVarNode, self).__init__(node)
        self.node = node

    def set_shape(self, shape):
        """
        Set the node variable shape.

        Args:
            shape(list): shape to be set.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
4344
            "The node variable description can not be None."
4345 4346 4347 4348 4349 4350 4351 4352 4353 4354
        self.node.var().set_shape(shape)

    def persistable(self):
        """
        If the variable node is a persistable variable, then return true.

        Returns:
            bool: indicate whether the variable is persistable.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
4355
            "The node variable description can not be None."
4356 4357
        return self.node.var().persistable()

4358 4359 4360 4361 4362 4363 4364 4365
    def type(self):
        """
        Return the variable type.

        Returns:
            core.VarDesc.VarType: the variable type.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
4366
            "The node variable description can not be None."
4367 4368 4369 4370 4371 4372 4373 4374 4375 4376
        return self.node.var().type()

    def dtype(self):
        """
        Return the variable data type.

        Returns:
            core.VarDesc.VarType: the variable data type.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
4377
            "The node variable description can not be None."
4378 4379 4380 4381 4382 4383 4384 4385 4386 4387
        return self.node.var().dtype()

    def shape(self):
        """
        Return the variable shape.

        Returns:
            list: the variable shape.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
4388
            "The node variable description can not be None."
4389 4390
        return self.node.var().shape()

4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrOpNode): node inputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrOpNode): node outputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.outputs]


class IrOpNode(IrNode):
    """
    Python IrOpNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrOpNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_op(), \
            'node must be the instance of core.Node and it must be a operator node.'
        super(IrOpNode, self).__init__(node)
        self.node = node

    def rename_input(self, old_input_name, new_input_name):
        """
        Rename the input of this node.

        Args:
            old_input_name(str): the old input name.
            new_input_name(str): the new input name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4438
            "The node operator description can not be None."
4439 4440
        self.node.op()._rename_input(old_input_name, new_input_name)

4441 4442 4443 4444 4445 4446 4447 4448 4449
    def rename_output(self, old_output_name, new_output_name):
        """
        Rename the output of this node.

        Args:
            old_output_name(str): the old output name.
            new_output_name(str): the new output name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4450
            "The node operator description can not be None."
4451 4452
        self.node.op()._rename_output(old_output_name, new_output_name)

4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463
    def input(self, name):
        """
        Get the argument name list by the parameter name for input.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4464
            "The node operator description can not be None."
4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477
        return self.node.op().input(name)

    def output(self, name):
        """
        Get the argument name list by the parameter name for output.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4478
            "The node operator description can not be None."
4479 4480 4481 4482 4483 4484 4485 4486 4487 4488
        return self.node.op().output(name)

    def set_type(self, new_type):
        """
        Change the operator type into new type.

        Args:
            new_type(str): new operator type to be set.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4489
            "The node operator description can not be None."
4490 4491
        return self.node.op().set_type(new_type)

4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506
    def set_attr(self, name, val):
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.
        """
        self._update_desc_attr(name, val)

    def _update_desc_attr(self, name, val):
        """
        Update the value of the op desc's attribute by attribute's name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4507
            "The node operator description can not be None."
4508
        desc = self.node.op()
4509 4510 4511 4512 4513
        if isinstance(val, Variable):
            desc.set_var_attr(name, val.desc)
        elif isinstance(val, list) and _all_is_type(val, Variable):
            desc.set_vars_attr(name, [v.desc for v in val])
        elif isinstance(val, Block):
4514
            desc.set_block_attr(name, val.desc)
4515
        elif isinstance(val, list) and val and _all_is_type(val, Block):
4516 4517
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
4518
                isinstance(val, core.ProgramDesc):
4519 4520 4521 4522
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)

4523 4524 4525 4526 4527 4528 4529 4530
    def input_arg_names(self):
        """
        Return input arguments' names of this op node.

        Returns:
            list(str): input arguments' names of this op node.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4531
            "The node operator description can not be None."
4532 4533 4534 4535 4536 4537 4538 4539 4540 4541
        return self.node.op().input_arg_names()

    def output_arg_names(self):
        """
        Return output arguments' names of this op node.

        Returns:
            list(str): output arguments' names of this op node.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
4542
            "The node operator description can not be None."
4543 4544
        return self.node.op().output_arg_names()

4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrVarNode): node inputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrVarNode): node outputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.outputs]


4566 4567
class IrGraph(object):
    """
4568
    Python IrGraph. Beneath it is a core.Graph, which is used for
4569
    creating a c++ Ir Pass Graph. An IrGraph is just a graph view of
4570 4571
    a Program. In an IrGraph, both Variables and Operators are graph
    nodes.
4572 4573 4574 4575
    """

    def __init__(self, graph, for_test=False):
        """
4576 4577
        Construct an IrGraph using core.Graph.

4578 4579 4580 4581 4582 4583 4584 4585 4586
        Args:
            graph(core.Graph): C++ Graph.
            for_test(bool): True for the test graph and false for the train graph.
        """
        assert isinstance(
            graph, core.Graph), 'graph must be the instance of core.Graph.'
        self.graph = graph
        self._for_test = for_test

4587 4588 4589 4590
    def clone(self):
        """
        Create a new and duplicated IrGraph.

4591 4592 4593
        Warns:
            The method only clones the graph structure, not its attributes.

4594 4595 4596
        Returns:
            IrGraph: A new and duplicated graph.
        """
4597
        g = self.graph.clone()
4598 4599
        return IrGraph(g, self._for_test)

4600
    def is_test(self):
4601 4602 4603
        """
        If the graph is used for testing, the function returns true. Otherwise, returns false.
        """
4604 4605
        return self._for_test

W
WangZhen 已提交
4606
    def all_nodes(self):
4607 4608 4609
        """
        Return all nodes included in the graph as a set.
        """
4610
        return {IrNode(node) for node in self.graph.nodes()}
4611

4612
    def all_var_nodes(self):
4613 4614 4615
        """
        Return all variable nodes included in the graph as a set.
        """
4616
        return {IrVarNode(node) for node in self.graph.nodes() if node.is_var()}
4617

4618
    def all_persistable_nodes(self):
4619 4620 4621
        """
        Return all persistable variable nodes included in the graph as a set.
        """
W
WangZhen 已提交
4622 4623 4624 4625 4626
        persistable_nodes = set()
        for node in self.graph.nodes():
            if node.is_var() and node.var() is not None and node.var(
            ).persistable():
                persistable_nodes.add(node)
4627
        return {IrVarNode(p) for p in persistable_nodes}
W
WangZhen 已提交
4628

4629
    def all_op_nodes(self):
4630 4631 4632
        """
        Return all operator nodes included in the graph as a set.
        """
4633
        return {IrOpNode(node) for node in self.graph.nodes() if node.is_op()}
4634

4635 4636 4637 4638 4639 4640
    def all_sub_graphs(self, for_test=False):
        """
        Return all sub_graphs included in the main graph as a set.
        """

        return [
4641
            IrGraph(self.graph.get_sub_graph(i), for_test=for_test)
4642 4643 4644 4645 4646 4647 4648 4649 4650
            for i in range(self.graph.sub_graph_size())
        ]

    def get_sub_graph(self, i, for_test=False):
        """
        Return i-th sub_graph in the main graph.
        """
        return IrGraph(self.graph.get_sub_graph(i), for_test=for_test)

4651
    def create_persistable_node(self, name, var_type, shape, var_dtype):
4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662
        """
        Create a persistable variable node in the graph. In IrGraph,
        it can not distinguish between persistable variables and parameters.

        Args:
            name(str): the name of the persistable variable node.
            vart_type(core.VarDesc.VarType): the type of the persistable variable node.
            shape(list): the shape of the persistable variable node.
            var_dtype(core.VarDesc.VarType): the data type of the persistable variable node.

        Returns:
4663
            IrVarNode: the created persistable variable node.
4664
        """
4665 4666 4667 4668 4669
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
        var_desc.set_persistable(True)
4670
        return IrVarNode(self.graph.create_var_node(var_desc))
4671 4672

    def create_var_node(self, name, var_type, shape, var_dtype):
4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683
        """
        Create a variable node in the graph. The created variable node is
        not persistable.

        Args:
            name(str): the name of the variable node.
            vart_type(core.VarDesc.VarType): the type of the variable node.
            shape(list): the shape of the variable node.
            var_dtype(core.VarDesc.VarType): the data type of the variable node.

        Returns:
4684
            IrVarNode: the created variable node.
4685 4686
        """

4687 4688 4689 4690
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
4691
        return IrVarNode(self.graph.create_var_node(var_desc))
4692

4693 4694 4695 4696 4697 4698
    def create_control_dep_var(self):
        """
        create a control var
        """
        return IrVarNode(self.graph.create_control_dep_var())

4699
    def create_var_node_from_desc(self, var_desc):
4700 4701 4702 4703 4704 4705 4706 4707
        """
        Create a variable node by using an existing VarDesc in the graph.
        Depend on the giving VarDesc, the created variable node may be persistable.

        Args:
            var_desc(core.VarDesc): the giving variable description.

        Returns:
4708
            IrVarNode: the created variable node.
4709
        """
4710
        return IrVarNode(self.graph.create_var_node(var_desc))
4711 4712

    def create_op_node(self, op_type, attrs, inputs, outputs):
4713 4714 4715 4716 4717 4718 4719
        """
        Create a operator node in the graph.

        Args:
            op_type(str): the type of the operator node.
            attrs(dict): the attributes of the operator node.
            inputs(dict): the inputs of the operator node.
T
tianshuo78520a 已提交
4720
            outputs(dict): the outputs of the operator node.
4721 4722

        Returns:
4723
            IrOpNode: the created operator node.
4724
        """
4725 4726
        op_desc = core.OpDesc()
        op_desc.set_type(op_type)
4727
        for attr, value in six.iteritems(attrs):
4728
            self._update_desc_attr(op_desc, attr, value)
4729
        for input_name, var_nodes in six.iteritems(inputs):
4730 4731 4732 4733
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_input(input_name,
                              [var_node.name() for var_node in var_nodes])
4734
        for output_name, var_nodes in six.iteritems(outputs):
4735 4736 4737 4738
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_output(output_name,
                               [var_node.name() for var_node in var_nodes])
4739
        return IrOpNode(self.graph.create_op_node(op_desc))
4740 4741

    def create_op_node_from_desc(self, op_desc):
4742 4743 4744 4745 4746 4747 4748
        """
        Create a operator node by using an existing OpDesc in the graph.

        Args:
            op_desc(core.VarDesc): the giving operator description.

        Returns:
4749
            IrOpNode: the created operator node.
4750
        """
4751
        return IrOpNode(self.graph.create_op_node(op_desc))
4752 4753

    def update_input_link(self, old_input_node, new_input_node, op_node):
4754 4755 4756 4757
        """
        Update the input's link of a operator node.

        Args:
4758 4759 4760
            old_input_node(IrNode): the old input node of the giving op_node.
            new_input_node(IrNode): the new input node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
4761
        """
4762
        assert old_input_node.node in self.graph.nodes() and new_input_node.node in \
T
tangwei12 已提交
4763
            self.graph.nodes() and op_node.node in self.graph.nodes(), \
4764
            'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.'
4765 4766 4767 4768
        old_input_node.remove_output(op_node)
        op_node.remove_input(old_input_node)
        new_input_node.append_output(op_node)
        op_node.append_input(new_input_node)
4769
        op_node.rename_input(old_input_node.name(), new_input_node.name())
4770

4771 4772 4773 4774 4775 4776 4777 4778 4779 4780
    def update_output_link(self, old_output_node, new_output_node, op_node):
        """
        Update the output's link of an operator node.

        Args:
            old_output_node(IrNode): the old output node of the giving op_node.
            new_output_node(IrNode): the new output node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
        """
        assert old_output_node.node in self.graph.nodes() and new_output_node.node in \
T
tangwei12 已提交
4781
            self.graph.nodes() and op_node.node in self.graph.nodes(), \
4782
            'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.'
4783 4784 4785 4786 4787 4788
        old_output_node.remove_input(op_node)
        op_node.remove_output(old_output_node)
        new_output_node.append_input(op_node)
        op_node.append_output(new_output_node)
        op_node.rename_output(old_output_node.name(), new_output_node.name())

4789
    def link_to(self, node_in, node_out):
4790 4791 4792 4793
        """
        Connect two nodes.

        Args:
4794 4795
            node_in(IrNode): the input node.
            node_out(IrNode): the output node.
4796
        """
4797 4798 4799 4800
        assert node_in.node in self.graph.nodes(), (
            'node_in(%s) must be in the graph nodes.' % node_in.node.name())
        assert node_out.node in self.graph.nodes(), (
            'node_out(%s) must be in the graph nodes.' % node_out.node.name())
4801 4802
        node_in.append_output(node_out)
        node_out.append_input(node_in)
4803 4804

    def safe_remove_nodes(self, remove_nodes):
4805 4806 4807 4808 4809 4810 4811
        """
        Remove nodes safely since links connected to these removed nodes are
        also removed.

        Args:
            remove_nodes(set): the nodes prepared to be removed.
        """
4812
        if not isinstance(remove_nodes, set):
W
WangZhen 已提交
4813 4814 4815 4816
            if isinstance(remove_nodes, Iterable):
                remove_nodes = set(remove_nodes)
            else:
                remove_nodes = {remove_nodes}
4817 4818
        original_nodes = {n.node for n in remove_nodes}
        core.graph_safe_remove_nodes(self.graph, original_nodes)
4819

Z
Zhen Wang 已提交
4820 4821 4822 4823 4824 4825 4826 4827
    def resolve_hazard(self):
        ordered_nodes = core.topology_sort(self.graph)
        var_nodes = dict()
        for node in ordered_nodes:
            if node.is_op() and node.op() is not None:
                for each_var_name in node.op().input_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
4828
                            self._find_node_by_name(node.inputs, each_var_name)
Z
Zhen Wang 已提交
4829 4830 4831 4832
                        ]
                for each_var_name in node.op().output_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
4833
                            self._find_node_by_name(node.outputs, each_var_name)
Z
Zhen Wang 已提交
4834 4835 4836
                        ]
                    else:
                        var_nodes[each_var_name].append(
4837 4838
                            self._find_node_by_name(node.outputs,
                                                    each_var_name))
Z
Zhen Wang 已提交
4839 4840
        self.graph.resolve_hazard(var_nodes)

W
WangZhen 已提交
4841
    def has_circle(self):
4842 4843 4844 4845 4846 4847
        """
        Check if the graph has a circle.

        Returns:
            bool: True if the graph has a circle else False.
        """
W
WangZhen 已提交
4848 4849 4850
        return core.has_circle(self.graph)

    def graph_num(self):
4851 4852 4853 4854 4855 4856
        """
        Count the number of unconnected graphs in this graph.

        Returns:
            int: the number of unconnected graphs.
        """
W
WangZhen 已提交
4857 4858 4859
        return core.graph_num(self.graph)

    def topology_sort(self):
4860 4861 4862
        """
        Perform the topology sort operation on the graph.

T
tianshuo78520a 已提交
4863
        Notes: the `graph` can not contain a circle.
4864 4865

        Returns:
Z
Zhen Wang 已提交
4866
            list(IrNode): nodes in topology order.
4867
        """
4868
        ordered_nodes = core.topology_sort(self.graph)
Z
Zhen Wang 已提交
4869
        return [IrNode(n) for n in ordered_nodes]
W
WangZhen 已提交
4870 4871

    def build_adjacency_list(self):
4872 4873 4874 4875
        """
        Build an adjacency list of operations for the `graph`.

        Returns:
4876
            dict{IrNode: set(IrNode)}: the adjacency list.
4877
        """
4878 4879 4880 4881 4882
        adj_list = core.build_adjacency_list(self.graph)
        wrapped_adj_list = dict()
        for k, v in six.iteritems(adj_list):
            wrapped_adj_list[IrNode(k)] = {IrNode(n) for n in v}
        return wrapped_adj_list
W
WangZhen 已提交
4883

4884 4885 4886 4887 4888 4889 4890 4891
    def draw(self, save_path, name, marked_nodes=None, remove_ctr_var=True):
        """
        Draw the graph. If `dot` command is installed, the drawn graph
        will be saved as pdf file type, otherwise dot file type is used.

        Args:
            save_path(str): the save path of drawn graph.
            name(str): the name of drawn graph.
4892
            marked_nodes(set(IrNode)): nodes that are needed to be marked.
4893 4894 4895 4896 4897
            Default value is None.
            remove_ctr_var(bool): If it is set True, all control variable nodes
            in the graph will be removed. Default value is True.
        """

4898 4899
        def _convert_to_pdf(dot_file_path):
            pdf_save_path = os.path.splitext(dot_file_path)[0] + '.pdf'
4900 4901 4902
            exited_code = subprocess.call('dot -Tpdf ' + dot_file_path +
                                          ' -o ' + pdf_save_path,
                                          shell=True)
4903 4904 4905 4906 4907
            if exited_code != 0:
                print('The dot command is needed for creating pdf files.')
                print('The {} is saved as the dot filetype.'.format(
                    dot_file_path))

4908
        remove_ctr_vars = set()
4909
        if remove_ctr_var:
4910
            for node in self.all_var_nodes():
4911 4912 4913
                if node.is_ctrl_var():
                    remove_ctr_vars.add(node)
            self.safe_remove_nodes(remove_ctr_vars)
4914 4915
        print('Total ops num = {}.'.format(len(self.all_op_nodes())))

4916 4917
        if marked_nodes is not None:
            if not isinstance(marked_nodes, set):
4918 4919 4920 4921 4922 4923
                if isinstance(marked_nodes, Iterable):
                    marked_nodes = set(marked_nodes)
                else:
                    marked_nodes = {marked_nodes}
            marked_nodes = {n.node for n in marked_nodes}
            remove_ctr_vars = {n.node for n in remove_ctr_vars}
4924 4925 4926 4927
            marked_nodes = marked_nodes - remove_ctr_vars
            if self.graph.has('__graphviz__marked_node__'):
                self.graph.erase('__graphviz__marked_node__')
            self.graph.set('__graphviz__marked_node__', marked_nodes)
4928 4929
        if not os.path.exists(save_path):
            os.makedirs(save_path)
4930 4931 4932 4933 4934 4935 4936
        viz_dot_path = os.path.join(save_path, name) + '.dot'
        viz_pass = core.get_pass('graph_viz_pass')
        viz_pass.set('graph_viz_path', viz_dot_path)
        viz_pass.apply(self.graph)
        _convert_to_pdf(viz_dot_path)

    def to_program(self):
4937 4938 4939
        """
        Convert the graph into a Program.

Z
Zhen Wang 已提交
4940
        WARN: When the graph includes backward operator nodes, the
4941 4942 4943 4944 4945 4946
        conversion process may be failed. Usually, this function is
        only used to convert a test graph.

        Returns:
            Program: a program converted from the graph.
        """
4947
        convert_pass = core.get_pass('graph_to_program_pass')
4948 4949
        desc = core.ProgramDesc()
        convert_pass.set_not_owned('program', desc)
4950 4951 4952 4953
        convert_pass.apply(self.graph)
        program = Program._construct_from_desc(desc)
        return program

4954 4955 4956 4957 4958 4959 4960 4961
    def _find_node_by_name(self, nodes, node_name):
        """
        Find a node in the giving nodes set by the name.
        """
        target_node = None
        for n in nodes:
            if n.name() == node_name:
                target_node = n
4962 4963
        assert target_node is not None, (
            "Cannot find the target node (%s)in the giving set." % node_name)
4964 4965
        return target_node

4966 4967 4968 4969
    def _update_desc_attr(self, desc, name, val):
        """
        Update the value of desc's attribute by attribute's name.
        """
4970 4971 4972 4973 4974
        if isinstance(val, Variable):
            desc.set_var_attr(name, val.desc)
        elif isinstance(val, list) and _all_is_type(val, Variable):
            desc.set_vars_attr(name, [v.desc for v in val])
        elif isinstance(val, Block):
4975
            desc.set_block_attr(name, val.desc)
4976
        elif isinstance(val, list) and val and _all_is_type(val, Block):
4977 4978 4979 4980 4981 4982 4983 4984
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)


Y
Yu Yang 已提交
4985
class Program(object):
D
dzhwinter 已提交
4986
    """
4987
    Create Python Program.  It has at least one :ref:`api_guide_Block_en`, when the
4988
    control flow op like conditional_block, while :ref:`api_paddle_fluid_layers_While` is included,
J
Jiabin Yang 已提交
4989
    it will contain nested block.
4990

J
Jiabin Yang 已提交
4991 4992 4993
    Please reference the
    `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_
    for details.
D
dzhwinter 已提交
4994

J
Jiabin Yang 已提交
4995
    A set of Program usually contains startup program and main program.
J
Jiabin Yang 已提交
4996
    A startup program is set to contain some initial work, eg. initialize the ``Parameter``, and the main
J
Jiabin Yang 已提交
4997 4998 4999 5000 5001 5002 5003
    program will contain the network structure and vars for train.

    A set of Program can be used for test or train, in train program ,
    Paddle will contain all content to build a train network,  in test
    program Paddle will prune some content which is irrelevant to test, eg.
    backward ops and vars.

J
Jiabin Yang 已提交
5004
    **Notes**:
5005 5006 5007
        **we have** :ref:`api_paddle_fluid_framework_default_startup_program` **and** :ref:`api_paddle_fluid_framework_default_main_program`
        **by default, a pair of them will shared the parameters. The** :ref:`api_paddle_fluid_framework_default_startup_program` **only run once to initialize parameters,**
        :ref:`api_paddle_fluid_framework_default_main_program` **run in every mini batch and adjust the weights.**
D
dzhwinter 已提交
5008 5009

    Returns:
J
Jiabin Yang 已提交
5010
        Program: An empty Program.
D
dzhwinter 已提交
5011 5012

    Examples:
5013 5014
        .. code-block:: python

5015 5016 5017 5018
            import paddle
            import paddle.static as static

            paddle.enable_static()
5019

5020 5021 5022 5023 5024
            main_program = static.Program()
            startup_program = static.Program()
            with static.program_guard(main_program=main_program, startup_program=startup_program):
                x = static.data(name="x", shape=[-1, 784], dtype='float32')
                y = static.data(name="y", shape=[-1, 1], dtype='int32')
5025
                z = static.nn.fc(name="fc", x=x, size=10, activation="relu")
5026 5027 5028

            print("main program is: {}".format(main_program))
            print("start up program is: {}".format(startup_program))
D
dzhwinter 已提交
5029 5030 5031

    """

5032 5033
    def __init__(self):
        self.desc = core.ProgramDesc()
Y
Yu Yang 已提交
5034 5035
        self.blocks = [Block(self, 0)]
        self.current_block_idx = 0
5036 5037
        global global_prog_seed
        self._seed = global_prog_seed
Y
yuyang18 已提交
5038
        self._current_role = core.op_proto_and_checker_maker.OpRole.Forward
5039
        self.__op_role_var = []
T
tangwei12 已提交
5040

5041 5042
        # for distribute training
        # _is_distributed = True if under distributed training
T
tangwei12 已提交
5043
        self._is_distributed = False
5044
        # _is_chief = True if the trainer is the first one, usually No.0
T
tangwei12 已提交
5045
        self._is_chief = False
5046 5047 5048
        # _parameters_on_pservers records all the parameters distributed on parameter servers.
        self._parameters_on_pservers = None
        # _endpoints is a list about parameter servers ip:port, such as ["ip:port","ip:port"]
T
tangwei12 已提交
5049
        self._endpoints = []
5050 5051 5052
        # if current role is parameter server, the _ps_endpoint is its "ip:port"
        self._ps_endpoint = None
        # trainers_endpoints, it is used for distribution.
5053
        self._trainers_endpoints = []
5054
        # the distributed lookup table names
T
tangwei12 已提交
5055
        self._distributed_lookup_table = None
5056 5057 5058

        # use Deep gradient comrepssion or not
        self._enable_dgc = False
5059 5060
        self._use_lamb = False

5061 5062 5063
        self._nccl_comm_num = 1
        self._use_hierarchical_allreduce = False
        self._hierarchical_allreduce_inter_nranks = 0
5064

5065 5066 5067
        # if this program has been optimized by distributed optimizer
        # fleet_opt will be given a value
        self._fleet_opt = None
D
dongdaxiang 已提交
5068
        self._program_config = None
5069

H
hutuxian 已提交
5070 5071 5072
        # assigned if this program has been parsed by a pipeline optimizer
        self._pipeline_opt = None

5073 5074 5075
        # assigned if this program has been parsed by a heter pipeline parameter server optimizer
        self._heter_pipeline_opt = None

5076 5077 5078
        # appending gradients times
        self._appending_grad_times = 0

5079 5080 5081 5082
        # identifier for auto checkpoint
        self._auto_checkpoint_name = unique_name.generate(
            "__auto_checkpoint_program__")

5083 5084
        # compiled program, i.e. Graph
        self._graph = None
5085 5086
        # to tag whether is startup_program
        self._is_start_up_program_ = False
5087

5088
    def _find_var_class_kwargs(self, new_desc):
5089 5090 5091 5092 5093 5094 5095 5096
        # NOTE: not all variables support shape/dtype/lod_level methods.
        # For example: RAW, STEP_SCOPES, etc.
        def get_var_desc_attr_or_none(var_desc, attr_name, allowed_types):
            if var_desc.type() in allowed_types:
                return getattr(var_desc, attr_name)()
            else:
                return None

5097 5098 5099 5100
        old_desc = self.desc
        all_new_vars = []
        block_num = new_desc.num_blocks()
        for idx in range(block_num):
5101 5102
            if (idx > (len(self.blocks) - 1)):
                self._create_block()
5103 5104 5105 5106 5107 5108 5109 5110 5111 5112
            new_block_desc = new_desc.block(idx)
            all_new_vars.append([])
            block_new_vars = all_new_vars[-1]
            for new_var_desc in new_block_desc.all_vars():
                if self.blocks[idx].has_var(new_var_desc.name()):
                    old_var = self.blocks[idx].var(new_var_desc.name())
                else:
                    old_var = None

                kwargs = {
5113 5114 5115 5116 5117 5118
                    'type':
                    new_var_desc.type(),
                    'name':
                    new_var_desc.name(),
                    'shape':
                    get_var_desc_attr_or_none(new_var_desc, "shape", [
5119 5120 5121 5122
                        core.VarDesc.VarType.LOD_TENSOR,
                        core.VarDesc.VarType.SELECTED_ROWS,
                        core.VarDesc.VarType.LOD_TENSOR_ARRAY,
                    ]),
5123 5124
                    'dtype':
                    get_var_desc_attr_or_none(new_var_desc, "dtype", [
5125 5126 5127 5128 5129 5130 5131 5132 5133
                        core.VarDesc.VarType.LOD_TENSOR,
                        core.VarDesc.VarType.SELECTED_ROWS,
                        core.VarDesc.VarType.LOD_TENSOR_ARRAY,
                    ]),
                    'lod_level':
                    get_var_desc_attr_or_none(new_var_desc, "lod_level", [
                        core.VarDesc.VarType.LOD_TENSOR,
                        core.VarDesc.VarType.LOD_TENSOR_ARRAY,
                    ]),
5134 5135 5136 5137 5138 5139 5140 5141 5142 5143
                    'error_clip':
                    old_var.error_clip if old_var is not None else None,
                    'stop_gradient':
                    old_var.stop_gradient if old_var is not None else False,
                    'is_data':
                    old_var.is_data if old_var is not None else False,
                    'need_check_feed':
                    new_var_desc.need_check_feed(),
                    'belong_to_optimizer':
                    old_var.belong_to_optimizer
5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173
                    if old_var is not None else False,
                }

                if isinstance(old_var, Parameter):
                    kwargs.update({
                        'trainable': old_var.trainable,
                        'optimize_attr': old_var.optimize_attr,
                        'regularizer': old_var.regularizer,
                        'do_model_average': old_var.do_model_average,
                        'need_clip': old_var.need_clip,
                        'is_distributed': old_var.is_distributed,
                        'is_parameter': old_var.is_parameter,
                    })
                    block_new_vars.append({
                        'class': Parameter,
                        'kwargs': copy.deepcopy(kwargs),
                    })
                else:
                    kwargs['persistable'] = new_var_desc.persistable()
                    block_new_vars.append({
                        'class': Variable,
                        'kwargs': copy.deepcopy(kwargs),
                    })

        return all_new_vars

    def _rebuild_from_desc(self, desc):
        all_new_vars = self._find_var_class_kwargs(desc)
        block_num = desc.num_blocks()
        assert block_num == len(all_new_vars)
5174
        assert block_num == self.desc.num_blocks()
5175 5176

        # clear old blocks and desc
5177 5178 5179 5180 5181 5182 5183 5184 5185
        for idx in range(block_num):
            block = self.blocks[idx]
            block.vars.clear()
            block.ops.clear()

        for idx in range(block_num):
            block_desc = self.blocks[idx].desc
            new_block_desc = desc.block(idx)
            block_desc._move_from(new_block_desc)
5186

5187
        del desc
5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206

        # add new vars first
        for idx in range(block_num):
            block = self.blocks[idx]
            for new_var in all_new_vars[idx]:
                clazz = new_var['class']
                kwargs = new_var['kwargs']
                kwargs['block'] = block
                clazz(**kwargs)

        # then append op
        for idx in range(block_num):
            block = self.blocks[idx]
            block_desc = self.desc.block(idx)
            for op_idx in range(block_desc.op_size()):
                op_desc = block_desc.op(op_idx)
                op = Operator(block=block, desc=op_desc)
                block.ops.append(op)

5207 5208 5209 5210 5211 5212 5213 5214 5215 5216
    def global_seed(self, seed=0):
        """
        Set global seed for Program

        Returns:
            None.

        Examples:
            .. code-block:: python

5217 5218
                import paddle
                import paddle.static as static
5219

5220 5221 5222
                paddle.enable_static()

                prog = static.default_main_program()
5223 5224 5225 5226 5227
                print(prog.random_seed)
                ## 0
                ## the default random seed is 0

                prog.global_seed(102)
5228
                prog1 = static.default_main_program()
5229 5230 5231 5232 5233 5234 5235 5236
                print(prog1.random_seed)
                ## 102
                ## the random seed is 102
        """
        global global_prog_seed
        global_prog_seed = seed
        self._seed = global_prog_seed

Y
yuyang18 已提交
5237
    @property
5238
    def _op_role(self):
Y
yuyang18 已提交
5239 5240 5241 5242 5243 5244 5245 5246
        """
        The operator role. In a enum {Forward, Backward, Optimize}.

        Notes: this is a low level API. It is used only for ParallelExecutor to
        duplicate or schedule operator to devices.

        For example, the forward operator should be executed on every device.
        The backward operator should be executed on every device and the
5247
        parameter gradient of backward (use :code:`_op_role_var` to get this
Y
yuyang18 已提交
5248 5249 5250 5251
        variable) operator should be merged to one device. The optimization
        operators should be executed on only one device and broadcast the
        optimization result, i.e., the new parameter, to every other device.
        """
Y
yuyang18 已提交
5252 5253
        return self._current_role

5254 5255
    @_op_role.setter
    def _op_role(self, role):
Y
yuyang18 已提交
5256 5257 5258
        self._current_role = role

    @property
5259
    def _op_role_var(self):
Y
yuyang18 已提交
5260
        """
5261
        The auxiliary variables for :code:`_op_role` property.
Y
yuyang18 已提交
5262

5263
        See Also: :code:`Program._op_role`'s documentation for details.
Y
yuyang18 已提交
5264 5265 5266

        Notes: This is a very low-level API. Users should not use it directly.
        """
5267
        return self.__op_role_var
Y
yuyang18 已提交
5268

5269
    @signature_safe_contextmanager
5270 5271 5272 5273 5274
    def _backward_role_guard(self):
        tmp_role = self._current_role

        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Backward
5275 5276 5277 5278
        try:
            yield
        finally:
            self._current_role = tmp_role
5279

S
rename  
sneaxiy 已提交
5280
    @signature_safe_contextmanager
W
Wu Yi 已提交
5281
    def _optimized_guard(self, param_and_grads):
Y
yuyang18 已提交
5282 5283 5284 5285 5286 5287 5288
        """
        A with guard to set :code:`Optimization` :code:`OpRole` and
        :code:`OpRoleVar` automatically.

        Notes: This is a very low level API. Users should not use it directly.

        Args:
5289
            param_and_grads(list): The variables (names) to be optimized.
Y
yuyang18 已提交
5290 5291 5292

        Examples:

5293
            >>> import paddle.fluid as fluid
Y
yuyang18 已提交
5294
            >>> p, g = backward(...)
W
Wu Yi 已提交
5295
            >>> with program._optimized_guard([p,g]):
Y
yuyang18 已提交
5296 5297
            >>>     p = p - 0.001 * g
        """
X
Xin Pan 已提交
5298
        tmp_role = self._current_role
5299
        tmp_var = self.__op_role_var
X
Xin Pan 已提交
5300

Y
yuyang18 已提交
5301 5302
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Optimize
5303
        self.__op_role_var = [
5304 5305 5306
            var.name if isinstance(var, Variable) else var
            for var in param_and_grads
        ]
5307 5308 5309 5310 5311
        try:
            yield
        finally:
            self.__op_role_var = tmp_var
            self._current_role = tmp_role
Y
Yu Yang 已提交
5312

S
rename  
sneaxiy 已提交
5313
    @signature_safe_contextmanager
X
Xin Pan 已提交
5314
    def _lr_schedule_guard(self, is_with_opt=False):
5315 5316 5317 5318 5319 5320 5321
        """
        A with guard to set :code:`LRSched` :code:`OpRole` and
        :code:`OpRoleVar` automatically. The :code:`OpRoleVar` is
        set to the target learning rate.

        Notes: This is a very low level API. Users should not use it directly.

X
Xin Pan 已提交
5322 5323 5324 5325
        Args:
            is_with_opt: Only set to true if these ops a in the middle
                 of a bunch of optimize ops so that it can be treated
                 correctly. For example, sgd->lr_op->sgd->lr_op->sgd.
5326 5327 5328

        Examples:

5329
            >>> import paddle.fluid as fluid
5330 5331 5332 5333
            >>> p, g = backward(...)
            >>> with program.lr_schedule_guard():
            >>>     lr = lr * decay
        """
5334 5335

        tmp_role = self._current_role
5336
        tmp_var = self.__op_role_var
5337

5338 5339
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.LRSched
X
Xin Pan 已提交
5340 5341
        if is_with_opt:
            self._current_role = int(OpRole.LRSched) | int(OpRole.Optimize)
5342
        # TODO(typhoonzero): how to set target learning rate var
5343
        self.__op_role_var = []
5344 5345 5346 5347 5348
        try:
            yield
        finally:
            self.__op_role_var = tmp_var
            self._current_role = tmp_role
5349

5350
    def __str__(self):
Y
yuyang18 已提交
5351 5352 5353 5354 5355 5356 5357 5358 5359
        """
        Get the protobuf debug string of this Program.

        Returns:
            (str): The protobuf debug string.

        Raises:
            ValueError: If any of required fields is not set.
        """
5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379
        return self._to_readable_code()

    def _to_readable_code(self, skip_op_callstack=True):
        """
        Get readable debug string of Program.

        .. note::
            If you want to get the debug string in protobuf format,
            please use :code:`to_string` method.

        Args:
            skip_op_callstack(bool): whether to skip parsing Operator's attribute
                op_callstack, default value is True

        Returns:
            string: The formatted Program string.

        Examples:
            .. code-block:: python

5380 5381
            import paddle
            import paddle.static as static
5382

5383 5384 5385
            paddle.enable_static()

            cur_program = static.Program()
5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396
            cur_block = cur_program.current_block()
            new_var = cur_block.create_var(name="X",
                                           shape=[-1, 23, 48],
                                           dtype='float32')
            new_op = cur_block.append_op(type="abs",
                                inputs={"X": [new_var]},
                                outputs={"Out": [new_var]})
            print(cur_program._to_readable_code())
        """
        assert isinstance(
            skip_op_callstack, bool
Z
zhangchunle 已提交
5397
        ), "skip_op_callstack parameter's type is error, expect bool, received {}".format(
5398 5399 5400 5401
            type(skip_op_callstack))
        program_str = ""
        for block in self.blocks:
            program_str += block._to_readable_code(skip_op_callstack)
5402
            program_str += '\n'
5403
        return program_str
Y
Yang Yang(Tony) 已提交
5404

F
fengjiayi 已提交
5405 5406 5407
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
Y
yuyang18 已提交
5408

J
Jiabin Yang 已提交
5409 5410 5411
        Args:

            throw_on_error (bool): raise Value error when any of required fields is not set.
F
fengjiayi 已提交
5412

J
Jiabin Yang 已提交
5413
            with_details (bool): True if more details about variables and parameters, e.g., :code:`trainable`, :code:`optimize_attr`, need to print.
Y
yuyang18 已提交
5414

H
haowang101779990 已提交
5415
        Returns:
J
Jiabin Yang 已提交
5416
            str: The debug string describe current Program.
Y
yuyang18 已提交
5417 5418

        Raises:
J
Jiabin Yang 已提交
5419
            ValueError: If any of required fields is not set and throw_on_error is True.
F
fengjiayi 已提交
5420

5421 5422 5423
        Examples:
            .. code-block:: python

5424 5425 5426 5427
                import paddle
                import paddle.static as static

                paddle.enable_static()
5428

5429 5430 5431
                prog = static.default_main_program()
                x = static.data(name="X", shape=[2,3], dtype="float32")
                pred = static.nn.fc(x, size=3)
5432
                prog_string = prog.to_string(throw_on_error=True, with_details=False)
5433
                prog_string_with_details = prog.to_string(throw_on_error=False, with_details=True)
T
tianshuo78520a 已提交
5434
                print("program string without detail: {}".format(prog_string))
5435
                print("program string with detail: {}".format(prog_string_with_details))
F
fengjiayi 已提交
5436
        """
5437 5438 5439 5440 5441 5442 5443 5444 5445
        assert isinstance(
            throw_on_error, bool
        ), "The type of throw_on_error parameter is wrong, expected bool, but received {}.".format(
            type(throw_on_error))
        assert isinstance(
            with_details, bool
        ), "The type of with_details parameter is wrong, expected bool, but received {}.".format(
            type(with_details))

F
fengjiayi 已提交
5446 5447 5448 5449 5450 5451
        if with_details:
            res_str = ""
            for block in self.blocks:
                res_str += block.to_string(throw_on_error, with_details)
        else:
            protostr = self.desc.serialize_to_string()
5452 5453
            proto = framework_pb2.ProgramDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
5454 5455
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
5456

W
Wu Yi 已提交
5457
    def _get_desc(self):
Y
yuyang18 已提交
5458 5459 5460 5461 5462 5463 5464
        """
        Get the C++ side of `ProgramDesc` object pointer. The C++ object is
        exposed by :code:`pybind`.

        Notes: This is a very low level API. Users should not use this API
        directly.
        """
5465 5466
        return self.desc

X
version  
Xin Pan 已提交
5467 5468 5469
    def _version(self):
        return self.desc._version()

5470
    def clone(self, for_test=False):
Y
yuyang18 已提交
5471
        """
5472
        .. note:::
5473 5474
            1. :code:`Program.clone()` method DOES NOT clone :ref:`api_paddle_io_DataLoader` .
            2. Recommend you to use :code:`clone` before using :code:`Opimizer.minimize` .
5475
            3. This API has no effect in Dygraph Mode.
Y
yuyang18 已提交
5476

5477
        Create a new Program with forward content of original one when ``for_test=True``.
5478
        Create a new Program as same as the original one when ``for_test=False``.
5479

5480
        Some operators, e.g., :ref:`api_paddle_fluid_layers_batch_norm` , behave differently between
Y
yuyang18 已提交
5481 5482 5483
        training and testing. They have an attribute, :code:`is_test`, to
        control this behaviour. This method will change the :code:`is_test`
        attribute of them to :code:`True` when :code:`for_test=True`.
5484

5485 5486
        * Set for_test to False when you want to clone the program for training.
        * Set for_test to True when you want to clone the program for testing.
5487 5488
          We will prune the backward and optimize part of the program when you
          use :code:`clone` after :code:`Opimizer.minimize`, but we still
J
Jiabin Yang 已提交
5489
          recommend you to use :code:`clone` before using :code:`Opimizer.minimize`.
Y
yuyang18 已提交
5490

J
Jiabin Yang 已提交
5491
        For Example:
5492
          ::
L
Luo Tao 已提交
5493

5494 5495 5496 5497 5498 5499
            import paddle
            import paddle.static as static

            paddle.enable_static()

            img = static.data(name='image', shape=[None, 784])
5500
            pred = static.nn.fc(x=img, size=10, actvation='relu')
5501
            loss = paddle.mean(pred)
5502
            # Here we use clone before Momentum
5503 5504
            test_program = static.default_main_program().clone(for_test=True)
            optimizer = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
5505
            optimizer.minimize(loss)
5506

J
Jiabin Yang 已提交
5507
        Args:
5508

5509 5510
            for_test (bool): True if change the :code:`is_test` attribute of operators to :code:`True`
                and prune the backward and optimize part of the program. The default value is :code:`False` .
5511

J
Jiabin Yang 已提交
5512
        Returns:
5513
            Program: A new Program with forward content of original one when ``for_test=True``.  A new Program as same as the original one when ``for_test=False``
5514

Y
yuyang18 已提交
5515 5516 5517

        Examples:

5518 5519 5520 5521 5522 5523 5524
            .. note::
                The Program's order maybe different after :code:`clone` and
                this will not affect your training or testing progress. In the following
                example we give you an simple method :code:`print_prog(program)` to
                print Program Descs inorder to make sure you have same print result
                after :code:`clone`:

5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540
            .. code-block:: python

                import six

                def print_prog(prog):
                    for name, value in sorted(six.iteritems(prog.block(0).vars)):
                        print(value)
                    for op in prog.block(0).ops:
                        print("op type is {}".format(op.type))
                        print("op inputs are {}".format(op.input_arg_names))
                        print("op outputs are {}".format(op.output_arg_names))
                        for key, value in sorted(six.iteritems(op.all_attrs())):
                            if key not in ['op_callstack', 'op_role_var']:
                                print(" [ attrs: {}:   {} ]".format(key, value))


5541
            1. To clone a test program, the sample code is:
5542 5543 5544
                .. code-block:: python

                    import six
5545 5546 5547 5548 5549 5550
                    import paddle
                    import paddle.static as static
                    import paddle.utils as utils
                    import paddle.nn.functional as F

                    paddle.enable_static()
5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))

5563 5564
                    train_program = static.Program()
                    startup_program = static.Program()
J
Jiabin Yang 已提交
5565 5566 5567

                    # startup_program is used to do some parameter init work,
                    # and main program is used to hold the network
5568 5569 5570
                    with static.program_guard(train_program, startup_program):
                        with utils.unique_name.guard():
                            img = static.data(name='image', shape=[None, 784])
5571
                            hidden = static.nn.fc(x=img, size=200, activation='relu')
5572 5573
                            hidden = F.dropout(hidden, p=0.5)
                            loss = F.cross_entropy(
5574
                                input=static.nn.fc(x=hidden, size=10, activation='softmax'),
5575 5576
                                label=static.data(name='label', shape=[1], dtype='int64'))
                            avg_loss = paddle.mean(loss)
5577
                            test_program = train_program.clone(for_test=True)
5578
                    print_prog(test_program)
J
Jiabin Yang 已提交
5579 5580 5581 5582

                    # Due to parameter sharing usage for train and test, so we need to use startup program of train
                    # instead of using test startup program, while nothing is in test's startup program

5583
                    # In Paddle we will share weights by using the same Tensor name. In train and test program
J
Jiabin Yang 已提交
5584 5585 5586 5587
                    # all parameters will have the same name and this can make train and test program sharing parameters,
                    # that's why we need to use startup program of train. And for startup program of test, it has nothing,
                    # since it is a new program.

5588 5589 5590
                    with static.program_guard(train_program, startup_program):
                        with utils.unique_name.guard():
                            sgd = paddle.optimizer.SGD(learning_rate=1e-3)
5591 5592 5593
                            sgd.minimize(avg_loss)


5594
            2. The clone method can be avoid if you create program for training and program for testing individually.
5595 5596 5597
                .. code-block:: python

                    import six
5598 5599 5600 5601 5602 5603
                    import paddle
                    import paddle.static as static
                    import paddle.utils as utils
                    import paddle.nn.functional as F

                    paddle.enable_static()
5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))
5615

5616
                    def network():
5617
                        img = static.data(name='image', shape=[None, 784])
5618
                        hidden = static.nn.fc(x=img, size=200, activation='relu')
5619 5620
                        hidden = F.dropout(hidden, p=0.5)
                        loss = F.cross_entropy(
5621
                            input=static.nn.fc(x=hidden, size=10, activation='softmax'),
5622 5623
                            label=static.data(name='label', shape=[1], dtype='int64'))
                        avg_loss = paddle.mean(loss)
5624 5625
                        return avg_loss

5626 5627 5628 5629 5630
                    train_program_2 = static.Program()
                    startup_program_2 = static.Program()
                    test_program_2 = static.Program()
                    with static.program_guard(train_program_2, startup_program_2):
                        with utils.unique_name.guard():
5631
                            avg_loss = network()
5632
                            sgd = paddle.optimizer.SGD(learning_rate=1e-3)
5633
                            sgd.minimize(avg_loss)
5634
                    # the test startup program is not used.
5635 5636
                    with static.program_guard(test_program_2, startup_program_2):
                        with utils.unique_name.guard():
5637 5638
                            avg_loss = network()
                    print_prog(test_program_2)
5639

5640
            The two code snippets above will generate and print same programs.
5641
        """
5642

T
tangwei12 已提交
5643
        # NOTE(zhiqiu): we sync the original program first, since its program may diff with
5644 5645 5646
        # its desc due to modifying desc in c++ space. E.g. save op will add kLookupTablePath in desc.
        self._sync_with_cpp()

5647
        pruned_origin_block_id_map = None
5648
        if for_test:
5649 5650 5651 5652 5653 5654 5655 5656 5657
            forward_prog = Program()
            forward_prog.desc, pruned_origin_block_id_map = core.prune_backward(
                self.desc)
            forward_prog.blocks = [
                Block(forward_prog, i)
                for i in six.moves.range(forward_prog.desc.num_blocks())
            ]
            forward_prog._sync_with_cpp()
            p = forward_prog._inference_optimize(prune_read_op=False)
5658
        else:
5659
            p = Program()
G
gongweibao 已提交
5660 5661
            p.current_block_idx = self.current_block_idx
            p._seed = self._seed
5662
            p.desc = core.ProgramDesc(self.desc)
M
minqiyang 已提交
5663 5664 5665
            p.blocks = [
                Block(p, i) for i in six.moves.range(self.desc.num_blocks())
            ]
G
gongweibao 已提交
5666 5667

            p._current_role = self._current_role
5668
            p.__op_role_var = self.__op_role_var
5669
            p._appending_grad_times = self._appending_grad_times
5670 5671
            if hasattr(self, 'lr_sheduler'):
                p.lr_sheduler = self.lr_sheduler
G
gongweibao 已提交
5672

T
tangwei12 已提交
5673
            # NOTE(zhiqiu): we sync the cloned program, to update its program by
5674
            # its desc.
W
Wu Yi 已提交
5675
            p._sync_with_cpp()
5676

W
Wu Yi 已提交
5677
        p._copy_param_info_from(self)
5678
        p._copy_data_info_from(self, pruned_origin_block_id_map)
5679
        p._copy_dist_param_info_from(self)
Y
Yu Yang 已提交
5680
        return p
5681

5682
    def _prune(self, targets):
Y
yuyang18 已提交
5683 5684 5685 5686 5687 5688 5689 5690
        """
        Prune operators and variables which are not needed to generate
        :code:`targets`.

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
5691
            targets(list|Variable|Operator): A list of variables, operators, or variable names
Y
yuyang18 已提交
5692 5693 5694 5695
                need to be pruned

        Returns:
            Program:  A new, pruned program.
5696
        """
5697
        return self._prune_with_input([], targets)
5698 5699

    def _prune_with_input(self, feeded_var_names, targets):
Y
yuyang18 已提交
5700
        """
5701
        Prune operators and variables which are not needed to generate
5702 5703
        :code:`targets`. Prune operators and variables which are needed
        to generate feeded_var
5704 5705 5706 5707 5708 5709 5710

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            feeded_var_names(list|str): A list of variable names from where
                pruning start. If it is set as [], this API works just like _prune()
5711
            targets(list|Variable|Operator): A list of variables, operators, or variable names
5712 5713 5714 5715 5716 5717
                need to be pruned

        Returns:
            Program:  A new, pruned program.
        """

T
tangwei12 已提交
5718
        # NOTE(zhiqiu): we sync the original program first, since its program may diff with
5719 5720 5721
        # its desc due to modifying desc in c++ space. E.g. save op will add kLookupTablePath in desc.
        self._sync_with_cpp()

5722 5723
        if not isinstance(feeded_var_names, list):
            feeded_var_names = [feeded_var_names]
5724 5725
        if not isinstance(targets, list):
            targets = [targets]
5726 5727 5728

        for var in feeded_var_names:
            if not isinstance(var, six.string_types):
5729 5730 5731
                raise ValueError(
                    "All feeded_var_names of Program._prune_with_input() can only be "
                    "str, but received %s." % type(var))
5732

5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748
        # find out all variables that can be generated or updated with given feed
        generatable_vars = set()

        for idx, op in enumerate(self.global_block().ops):
            runnable_op = True
            for name in op.input_arg_names:
                if not self.global_block().has_var(name):
                    continue
                if self.global_block().var(name).persistable:
                    continue
                if name not in generatable_vars.union(feeded_var_names):
                    runnable_op = False
                    break
            if runnable_op:
                generatable_vars = generatable_vars.union(op.output_arg_names)

5749 5750 5751 5752
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
5753 5754 5755
                    name = t.name
                elif isinstance(t, six.string_types):
                    name = str(t)
5756
                else:
5757 5758 5759
                    raise ValueError(
                        "All targets of Program._prune_with_input() can only be "
                        "Variable or Operator, but received %s." % type(t))
5760 5761 5762 5763 5764 5765

                # NOTEZ(zhiqiu): For variable to be fed in fetch_list, there two cases:
                # (1) the variable is leaf, it has no op that generates it;
                # (2) the variable is not leaf, and we need to prune the op that generates it.
                # In both cases, wo can just skip target_op of that it.
                if name in feeded_var_names:
5766 5767 5768
                    # however if the var is also updated by a runnable op, will shall keep it
                    if name not in generatable_vars:
                        continue
5769

5770 5771 5772 5773 5774 5775 5776 5777 5778
                # After transpiler processing, the op that output this
                # variable maybe has been changed, so t.op is not reliable
                # and we need to find the current op that generate this
                # variable here.
                target_op = None
                global_block = self.global_block()
                for idx, op in enumerate(global_block.ops):
                    if name in op.output_arg_names:
                        # NOTE(zhiqiu): Find op that generate target name.
T
tangwei12 已提交
5779
                        # Skip optimize op except for optimize op in targets,
5780 5781 5782 5783 5784
                        # since optimize op generates parameters.
                        if op._is_optimize_op() and op not in targets:
                            continue
                        else:
                            target_op = op
5785

5786
                if target_op is not None:
5787 5788 5789
                    targets_idx.append([target_op.block.idx, target_op.idx])
            else:
                targets_idx.append([t.block.idx, t.idx])
5790

5791
        res = Program()
5792 5793
        res.desc, pruned_origin_block_id_map = core.prune(
            self.desc, set(feeded_var_names), targets_idx)
M
minqiyang 已提交
5794 5795 5796
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
5797
        res._sync_with_cpp()
5798 5799 5800 5801 5802

        res._copy_param_info_from(self)
        res._copy_data_info_from(self, pruned_origin_block_id_map)
        res._copy_dist_param_info_from(self)

5803 5804
        return res

X
Xin Pan 已提交
5805
    def _inference_optimize(self, prune_read_op=True):
Y
yuyang18 已提交
5806
        """
F
fengjiayi 已提交
5807 5808 5809 5810 5811
        This method will create a new program and do following adjustments on it:
        1. Remove all reader variables and their creator ops if exist.

        2. Remove the :code:`read_op` if exists.

5812
        3. change the :code:`is_test`
Y
yuyang18 已提交
5813 5814 5815
        attribute of operators to :code:`True`. All the :code:`Parameter`
        information will be lost.

5816
        Args:
X
Xin Pan 已提交
5817 5818
            prune_read_op(bool): remove the read ops that are added by py_reader
                                 for cpp inference library
5819

Y
yuyang18 已提交
5820 5821 5822 5823 5824 5825
        Notes: This API is a very low level API. Use
        :code:`Program.clone(for_test=True)` instead.

        Returns:
            Program: The new program.
        """
5826
        res = Program()
5827
        res.desc = core.ProgramDesc(self.desc)
F
fengjiayi 已提交
5828 5829 5830 5831

        # remove all readers and the read_op if exist
        read_op_idx = 0
        root_block = res.desc.block(0)
X
Xin Pan 已提交
5832
        if prune_read_op:
5833 5834 5835 5836 5837 5838 5839 5840 5841
            while True:
                if read_op_idx >= root_block.op_size() or root_block.op(
                        read_op_idx).type() == 'read':
                    break
                read_op_idx += 1
            if read_op_idx < root_block.op_size():
                root_block._remove_op(0, read_op_idx + 1)
            for var in root_block.all_vars():
                if var.type() == core.VarDesc.VarType.READER:
5842
                    root_block._remove_var(var.name().encode())
F
fengjiayi 已提交
5843 5844

        # change all `is_test` attributes to True
M
minqiyang 已提交
5845
        for i in six.moves.range(res.desc.num_blocks()):
5846
            block = res.desc.block(i)
M
minqiyang 已提交
5847
            for j in six.moves.range(block.op_size()):
5848 5849
                op = block.op(j)
                if op.has_attr('is_test'):
W
Wu Yi 已提交
5850
                    op._set_attr('is_test', True)
5851 5852 5853
                if op.type() == "batch_norm":
                    # Remove the output ReserveSpace of batch_norm if exists.
                    op.remove_output("ReserveSpace")
M
minqiyang 已提交
5854 5855 5856
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
5857
        res._sync_with_cpp()
5858 5859
        return res

5860
    def _remove_training_info(self, clip_extra=True):
5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879
        """
        This method will create a new program and do following adjustments on it:
        1. Remove all variable's `is_parameter` attribute if exist.

        2. Remove all variable's `stop_gradient` attribute if exist.

        Notes: This API is a very low level API.

        Returns:
            Program: The new program.
        """
        res = Program()
        res.desc = core.ProgramDesc(self.desc)

        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
        res._sync_with_cpp()

5880 5881
        # Note: The op_role and op_role_var cann't be deleted currently,
        # and we will try to remove them in the future.
5882
        common_clipped_attrs_list = ['op_callstack', 'with_quant_attr']
5883

5884 5885 5886 5887 5888
        for i in six.moves.range(res.desc.num_blocks()):
            block = res.desc.block(i)
            for var in block.all_vars():
                var.clear_is_parameter()
                var.clear_stop_gradient()
5889 5890
            if not clip_extra:
                continue
5891 5892 5893 5894
            for op_idx in range(0, block.op_size()):
                op = block.op(op_idx)
                if op.type() not in OpProtoHolder.instance().op_proto_map:
                    continue
5895 5896 5897

                extra_attrs_map = core.get_op_extra_attrs(op.type())

5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910
                proto = OpProtoHolder.instance().get_op_proto(op.type())
                remove_input_list = []
                for name in op.input_names():
                    find = False
                    for input_proto in proto.inputs:
                        if input_proto.name != name:
                            continue
                        if input_proto.extra:
                            remove_input_list.append(name)
                        find = True
                        break
                    if not find:
                        remove_input_list.append(name)
5911 5912 5913
                # The extra input of op will be removed in the future
                # for name in remove_input_list:
                #     op.remove_input(name)
5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926

                remove_output_list = []
                for name in op.output_names():
                    find = False
                    for output_proto in proto.outputs:
                        if output_proto.name != name:
                            continue
                        if output_proto.extra:
                            remove_output_list.append(name)
                        find = True
                        break
                    if not find:
                        remove_output_list.append(name)
5927 5928 5929
                # The extra output of op will be removed in the future
                # for name in remove_output_list:
                #     op.remove_output(name)
5930 5931 5932 5933 5934 5935 5936 5937 5938 5939

                op_quant_name = core.op_proto_and_checker_maker.kOpWithQuantAttrName(
                )
                quant = bool(op.attr(op_quant_name)
                             ) if op_quant_name in op.attr_names() else False
                quant_attrs = [
                    op_quant_name, "quantization_type", "skip_quant",
                    "activation_bits", "bit_length", "quantize_weight_bits",
                    "weight_quant_scale"
                ]
5940 5941
                for extra_attr_name in extra_attrs_map.keys():
                    op.remove_attr(extra_attr_name)
5942
                remove_attr_list = []
5943 5944 5945 5946 5947 5948
                for name in op.attr_names():
                    if quant:
                        if name in quant_attrs:
                            continue
                        if name.endswith("_threshold"):
                            continue
5949
                    if len(extra_attrs_map) > 0:
5950
                        if name in common_clipped_attrs_list:
5951
                            op.remove_attr(name)
5952
                        continue
5953 5954 5955 5956 5957 5958 5959 5960 5961 5962
                    find = False
                    for attr_proto in proto.attrs:
                        if attr_proto.name != name:
                            continue
                        find = True
                        break
                    if not find:
                        remove_attr_list.append(name)
                for name in remove_attr_list:
                    op.remove_attr(name)
5963 5964
        return res

5965 5966
    @staticmethod
    def parse_from_string(binary_str):
Y
yuyang18 已提交
5967
        """
5968
        .. note::
5969
            1. All information about parameters will be lost after serialization;
5970
            2. This API has no effect in Dygraph mode.
Y
yuyang18 已提交
5971

5972 5973
        Deserialize a Program from  `protobuf <https://en.wikipedia.org/wiki/Protocol_Buffers>`_  binary string.
        This method always use to save and load model
Y
yuyang18 已提交
5974

J
Jiabin Yang 已提交
5975
        Args:
Y
yuyang18 已提交
5976

J
Jiabin Yang 已提交
5977
            binary_str_type (str): the binary prootbuf string.
5978

J
Jiabin Yang 已提交
5979 5980
        Returns:
            Program: A deserialized Program.
5981 5982 5983 5984

        Examples:
            .. code-block:: python

5985 5986 5987 5988
                import paddle
                import paddle.static as static

                paddle.enable_static()
5989

5990 5991 5992 5993
                startup_prog = static.Program()
                main_prog = static.Program()
                with static.program_guard(startup_prog, main_prog):
                    x = static.data(name='X', shape=[1000, 784], dtype='float32')
5994

5995
                    y = static.data(name='Y', shape=[784, 100], dtype='float32')
5996

5997
                    z = paddle.matmul(x=x, y=y)
5998

5999 6000
                    binary_str = static.default_main_program().desc.serialize_to_string()
                    prog_restored = static.default_main_program().parse_from_string(binary_str)
6001

6002
                    print(static.default_main_program())
6003
                    print(prog_restored)
Y
yuyang18 已提交
6004
        """
6005 6006
        p = Program()
        p.desc = core.ProgramDesc(binary_str)
M
minqiyang 已提交
6007
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
W
Wu Yi 已提交
6008
        p._sync_with_cpp()
6009
        return p
Y
Yu Yang 已提交
6010

6011
    @staticmethod
6012
    def _construct_from_desc(desc):
6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027
        """
        Construct a program from program desc.

        Args:
            desc(core.ProgramDesc): The program desc for constructing.

        Returns:
            Program: A program.
        """
        p = Program()
        p.desc = desc
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
        p._sync_with_cpp()
        return p

D
dzhwinter 已提交
6028 6029
    @property
    def random_seed(self):
Y
yuyang18 已提交
6030
        """
J
Jiabin Yang 已提交
6031
        The default random seed for random operators in Program. ``0`` means get
Y
yuyang18 已提交
6032 6033
        the random seed from random device.

6034
        .. note::
6035
            It must be set before the operators have been added.
J
Jiabin Yang 已提交
6036 6037 6038

        Returns:
            int64: Random seed in current Program
6039

6040 6041 6042 6043

        Examples:
            .. code-block:: python

6044 6045 6046
                import paddle
                import paddle.static as static
                import paddle.nn.functional as F
6047

6048 6049 6050
                paddle.enable_static()

                prog = static.default_main_program()
6051
                random_seed = prog.random_seed
6052
                x_var = static.data(name="X", shape=[3,3], dtype="float32")
6053 6054 6055
                print(random_seed)
                ## 0
                ## the default random seed is 0
6056

6057
                # Here we need to set random seed before we use paddle.nn.functional.dropout
6058
                prog.random_seed = 1
6059
                z_var = F.dropout(x_var, 0.7)
6060

6061
                print(prog.random_seed)
6062 6063
                ## 1
                ## the random seed is change to 1
Y
yuyang18 已提交
6064
        """
D
dzhwinter 已提交
6065 6066
        return self._seed

Q
qiaolongfei 已提交
6067 6068
    @property
    def num_blocks(self):
Y
yuyang18 已提交
6069
        """
6070 6071
        The number of :ref:`api_guide_Block_en`  in this Program.

6072
        .. note::
6073
            This API has no effect in Dygraph mode.
J
Jiabin Yang 已提交
6074 6075 6076

        Returns:
            int(Platform-dependent size): num of :ref:`api_guide_Block_en`  in current Program
6077

6078 6079 6080 6081

        Examples:
            .. code-block:: python

6082 6083 6084 6085
                import paddle
                import paddle.static as static

                paddle.enable_static()
6086

6087
                prog = static.default_main_program()
6088 6089
                num_blocks = prog.num_blocks
                print(num_blocks)
6090

6091 6092
                # print result:
                # 1
Y
yuyang18 已提交
6093
        """
Q
qiaolongfei 已提交
6094 6095
        return self.desc.num_blocks()

D
dzhwinter 已提交
6096 6097 6098
    @random_seed.setter
    def random_seed(self, seed):
        if not isinstance(seed, int):
6099 6100 6101
            raise ValueError(
                "Program.random_seed's input seed must be an integer, but received %s."
                % type(seed))
D
dzhwinter 已提交
6102 6103
        self._seed = seed

Y
Yu Yang 已提交
6104
    def __repr__(self):
6105
        return self.__str__()
6106

Y
Yu Yang 已提交
6107
    def global_block(self):
Y
yuyang18 已提交
6108
        """
6109 6110
        .. note::
            This API has no effect in Dygraph mode.
6111 6112 6113

        Get the first :ref:`api_guide_Block_en` of this Program.

J
Jiabin Yang 已提交
6114 6115
        Returns:
            :ref:`api_guide_Block_en`: The first :ref:`api_guide_Block_en`  of this Program.
6116

6117 6118 6119 6120

        Examples:
            .. code-block:: python

6121 6122 6123 6124
                import paddle
                import paddle.static as static

                paddle.enable_static()
6125

6126
                prog = static.default_main_program()
6127 6128
                gb_block = prog.global_block()
                print(gb_block)
6129

Y
yuyang18 已提交
6130
        """
Y
Yu Yang 已提交
6131 6132
        return self.blocks[0]

Q
Qiao Longfei 已提交
6133
    def block(self, index):
Y
yuyang18 已提交
6134
        """
6135 6136
        .. note::
            This API has no effect in Dygraph mode.
Y
yuyang18 已提交
6137

6138 6139
        Get the :code:`index`  :ref:`api_guide_Block_en`  of this Program

J
Jiabin Yang 已提交
6140 6141
        Args:
            index (int) - The index of  :ref:`api_guide_Block_en`  to get
6142

J
Jiabin Yang 已提交
6143 6144
        Returns:
            :ref:`api_guide_Block_en`: The :code:`index` block
6145 6146 6147 6148

        Examples:
            .. code-block:: python

6149 6150 6151 6152
                import paddle
                import paddle.static as static

                paddle.enable_static()
6153

6154
                prog = static.default_main_program()
6155 6156
                block_0 = prog.block(0)
                print(block_0)
Y
yuyang18 已提交
6157
        """
Q
Qiao Longfei 已提交
6158 6159
        return self.blocks[index]

Y
Yu Yang 已提交
6160
    def current_block(self):
Y
yuyang18 已提交
6161
        """
6162 6163
        .. note::
            This API has no effect in Dygraph mode.
6164

J
Jiabin Yang 已提交
6165 6166
        Get the current  :ref:`api_guide_Block_en` . The :code:`current`  :ref:`api_guide_Block_en`
        is the  :ref:`api_guide_Block_en`  to append operators.
6167

J
Jiabin Yang 已提交
6168 6169
        Returns:
             :ref:`api_guide_Block_en`: The :code:`index`  :ref:`api_guide_Block_en`
6170

6171 6172 6173
        Examples:
            .. code-block:: python

6174 6175 6176 6177
                import paddle
                import paddle.static as static

                paddle.enable_static()
6178

6179
                prog = static.default_main_program()
6180 6181
                current_blk = prog.current_block()
                print(current_blk)
Y
yuyang18 已提交
6182
        """
Y
Yu Yang 已提交
6183 6184
        return self.blocks[self.current_block_idx]

W
Wu Yi 已提交
6185
    def _create_block(self, parent_idx=None):
Y
yuyang18 已提交
6186 6187 6188 6189 6190
        """
        Create a new block with the :code:`parent_idx` and change the current block
        to new block.

        Args:
J
Jiabin Yang 已提交
6191

Y
yuyang18 已提交
6192 6193 6194 6195 6196
            parent_idx(int): The parent block index.

        Returns:
            Block: The new block.
        """
Y
Yu Yang 已提交
6197
        new_block_idx = len(self.blocks)
F
update  
fengjiayi 已提交
6198 6199 6200
        parent = self.current_block() if parent_idx is None else self.block(
            parent_idx)
        self.desc.append_block(parent.desc)
Y
Yu Yang 已提交
6201 6202 6203 6204
        self.current_block_idx = new_block_idx
        self.blocks.append(Block(self, self.current_block_idx))
        return self.current_block()

W
Wu Yi 已提交
6205
    def _rollback(self):
Y
yuyang18 已提交
6206 6207 6208 6209 6210
        """
        Exit a code block, i.e., roll back to the parent block.
        Returns:
            None
        """
Y
Yu Yang 已提交
6211 6212
        self.current_block_idx = self.current_block().parent_idx

W
Wu Yi 已提交
6213
    def _sync_with_cpp(self):
Y
yuyang18 已提交
6214 6215 6216 6217 6218 6219 6220 6221 6222 6223
        """
        Synchronize Python instance to its binding C++ object instance.
        If the program is modified in C++ space, this method should be invoked.

        Notes: This is a very low level API. Users should not invoke it
        directly.

        Returns:
            None
        """
Q
Qiao Longfei 已提交
6224 6225 6226
        for block_idx in range(len(self.blocks), self.desc.num_blocks()):
            self.blocks.append(Block(self, block_idx))
        for block in self.blocks:
W
Wu Yi 已提交
6227
            block._sync_with_cpp()
Q
Qiao Longfei 已提交
6228

W
Wu Yi 已提交
6229
    def _copy_param_info_from(self, other):
6230
        """
6231
        Copy the information of parameters from other program.
D
dzhwinter 已提交
6232

Y
yuyang18 已提交
6233 6234 6235
        Notes: This is a very low level API. Users should not invoke it
        directly.

6236 6237 6238 6239 6240 6241 6242
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
6243 6244 6245
            raise TypeError(
                "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s"
                % type(other))
6246

W
Wu Yi 已提交
6247
        self.global_block()._copy_param_info_from(other.global_block())
6248

6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259
    def _copy_dist_param_info_from(self, other):
        """
        Copy the information of distributed information from other program.

        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
6260 6261 6262
            raise TypeError(
                "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s"
                % type(other))
6263 6264
        self._is_distributed = other._is_distributed
        self._is_chief = other._is_chief
6265
        self._parameters_on_pservers = other._parameters_on_pservers
6266
        self._endpoints = other._endpoints
6267
        self._ps_endpoint = other._ps_endpoint
6268 6269
        self._distributed_lookup_table = other._distributed_lookup_table

6270
    def _copy_data_info_from(self, other, pruned_origin_block_id_map=None):
F
fengjiayi 已提交
6271 6272
        """
        Copy the information of data variables from other program.
D
dzhwinter 已提交
6273

Y
yuyang18 已提交
6274 6275 6276
        Notes: This is a very low level API. Users should not invoke it
        directly.

F
fengjiayi 已提交
6277 6278
        Args:
            other(Program): Other program
6279
            pruned_origin_block_id_map(dict{int:int}): A dict which maps the block id in program
6280 6281
            self to the block id in program other. For example, {0:0, 1:1, 2:3} means block 0 in self is
            cloned from block 0 in other, etc. Default is None, which means default mapped,
6282
            {0:0, 1:1,..., n:n}.
F
fengjiayi 已提交
6283 6284 6285 6286 6287

        Returns:
            None
        """
        if not isinstance(other, Program):
6288 6289 6290
            raise TypeError(
                "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s"
                % type(other))
F
fengjiayi 已提交
6291

6292 6293 6294 6295 6296
        if not pruned_origin_block_id_map:
            pruned_origin_block_id_map = {
                i: i
                for i in six.moves.range(self.desc.num_blocks())
            }
6297 6298 6299

        # NOTE(zhiqiu): All vars in cloned program exist in original program.
        # The reverse is not true, due to backward pruning.
6300 6301
        for i, block in enumerate(self.blocks):
            other_block = other.blocks[pruned_origin_block_id_map[i]]
6302
            for var in list(block.vars.values()):
6303 6304 6305 6306 6307 6308 6309
                other_var = other_block.var(var.name)
                if other_var.is_data:
                    var.is_data = True
                if other_var.desc.need_check_feed():
                    var.desc.set_need_check_feed(True)
                if other_var.stop_gradient:
                    var.stop_gradient = True
F
fengjiayi 已提交
6310

6311
    def list_vars(self):
Y
yuyang18 已提交
6312
        """
6313
        Get all Tensors from this Program. A iterable object is returned.
Y
yuyang18 已提交
6314

J
Jiabin Yang 已提交
6315
        Returns:
6316
            iterable Tensors: The Generator will yield every Tensor in this program.
6317 6318 6319 6320

        Examples:
            .. code-block:: python

6321 6322
                import paddle
                import paddle.static as static
6323

6324 6325 6326 6327 6328
                paddle.enable_static()

                prog = static.default_main_program()
                img = static.data(name='img', shape=[None, 1,28,28], dtype='float32')
                label = static.data(name='label', shape=[None,1], dtype='int64')
6329 6330
                for var in prog.list_vars():
                    print(var)
T
tangwei12 已提交
6331

6332 6333
                # var img : LOD_TENSOR.shape(-1, 1, 28, 28).dtype(float32).stop_gradient(True)
                # var label : LOD_TENSOR.shape(-1, 1).dtype(int64).stop_gradient(True)
Y
yuyang18 已提交
6334
        """
6335
        for each_block in self.blocks:
6336
            for each_var in list(each_block.vars.values()):
6337 6338
                yield each_var

6339 6340 6341 6342 6343 6344 6345 6346 6347 6348
    def all_parameters(self):
        """
        Get all :ref:`api_guide_parameter_en` from this Program. A list object is returned.

        Returns:
            list[ :ref:`api_guide_parameter_en` ]: The list contians all parameters in this program.

        Examples:
            .. code-block:: python

6349 6350 6351 6352
                import paddle
                import paddle.static as static

                paddle.enable_static()
6353

6354 6355
                program = static.default_main_program()
                data = static.data(name='x', shape=[None, 13], dtype='float32')
6356
                hidden = static.nn.fc(x=data, size=10)
6357 6358
                loss = paddle.mean(hidden)
                paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
6359 6360 6361 6362 6363 6364 6365

                for param in program.all_parameters():
                    print(param)

                # Here will print all parameters in current program, in this example,
                # the result is like:
                #
6366 6367
                # persist trainable param fc_0.w_0 : LOD_TENSOR.shape(13, 10).dtype(float32).stop_gradient(False)
                # persist trainable param fc_0.b_0 : LOD_TENSOR.shape(10,).dtype(float32).stop_gradient(False)
6368 6369 6370 6371 6372 6373 6374 6375 6376 6377
                #
                # Here print(param) will print out all the properties of a parameter,
                # including name, type and persistable, you can access to specific
                # property of a parameter, such as param.name, param.type
        """
        parameters = []
        for each_block in self.blocks:
            parameters.extend(each_block.all_parameters())
        return parameters

6378 6379 6380 6381 6382 6383 6384 6385 6386
    def state_dict(self, mode='all', scope=None):
        """
        Get parameters and persistable buffers of program as a dict. The key is the name of the parameter or the name of the buffer.
        The value is the tensor of this variable in the given scope.

        .. note::
            This function MUST called after run start_up_program

        Args:
6387 6388 6389
            mode(str, optional): Source of the obtained parameters and buffers.
                    'opt' :  The return value only contains the variable in the optimizer.
                    'param' : The return value only contains the variable in the network, not the variable in the optimizer.
6390 6391
                    'all' : The return value contains the variable in the network and optimizer.
                    Default: 'all'
6392
            scope(Scope, optional) : If scope is None, state_dict will be set to global scope
6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419
                obtained through 'paddle.static.global_scope()'. Otherwise, value will be set to scope.
                Default: None

        Retruns:
            dict: a dict contains the parameters and persistable buffers.

        Examples:
            .. code-block:: python

                import paddle
                import paddle.static as static

                paddle.enable_static()

                x = static.data(name="x", shape=[10, 10], dtype='float32')
                y = static.nn.fc(x, 10)
                z = static.nn.fc(y, 10)

                place = paddle.CPUPlace()
                exe = static.Executor(place)
                exe.run(static.default_startup_program())
                prog = static.default_main_program()

                path = "./temp/model.pdparams"
                paddle.save(prog.state_dict(), path)
        """
        # The 'framework' is a low-level module, and 'executor'
6420
        # can not be imported at the begainning of this file.
6421 6422 6423 6424
        # Therefore, the above two modules are dynamically imported.
        from .executor import global_scope
        if scope is not None and not isinstance(scope, core._Scope):
            raise TypeError(
6425 6426
                "`scope` should be None or `paddle.static.Scope'` type, but received {}."
                .format(type(scope)))
6427 6428 6429 6430 6431

        if scope is None:
            scope = global_scope()

        if not isinstance(mode, str):
6432 6433 6434
            raise TypeError(
                "Type of `mode` should be string, but received {}.".format(
                    type(mode)))
6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460

        def is_parameter(var):
            return isinstance(var, Parameter)

        def is_persistable(var):
            if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \
                var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \
                var.desc.type() == core.VarDesc.VarType.READER:
                return False
            return var.persistable

        def is_belong_to_optimizer(var):
            if not (isinstance(var, Parameter) or var.desc.need_check_feed()):
                return is_persistable(var)
            return False

        def condition(var):

            if mode == 'param':
                return is_parameter(var)
            elif mode == 'opt':
                return is_belong_to_optimizer(var)
            elif mode == 'all':
                return is_parameter(var) or is_belong_to_optimizer(var)
            else:
                raise ValueError(
6461 6462
                    "`mode` string should be 'param', 'opt' or 'all', but received {}."
                    .format(mode))
6463 6464 6465 6466 6467 6468 6469 6470

        var_list = filter(condition, self.list_vars())

        state_dict = dict()
        for var in var_list:
            var_temp = scope.find_var(var.name)
            if var_temp is None:
                raise ValueError(
6471 6472
                    "Can not find Variable '{}' in the scope. Make sure it is initialized"
                    .format(var.name))
6473 6474 6475 6476 6477 6478
            state_dict[var.name] = var_temp.get_tensor()

        return state_dict

    def set_state_dict(self, state_dict, scope=None):
        """
6479
        Set parameters and persistable buffers in state_dict to program.
6480
        An exception will throw if shape or dtype of the parameters is not match.
6481

6482 6483 6484 6485
        .. note::
            This function MUST called after run start_up_program

        Args:
6486
            state_dict(dict): the dict store parameters and persistable buffers.
6487 6488
                The key is the name of the parameter or the name of the buffer.
                The value is the tensor of this variable in the given scope.
6489
            scope(Scope, optional) : If scope is None, state_dict will be set to global scope
6490 6491
                obtained through 'paddle.static.global_scope()'. Otherwise, value will be set to scope.
                Default: None
6492

6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541
        Returns:
            None

        Examples:
            .. code-block:: python

                import paddle
                import paddle.static as static

                paddle.enable_static()

                x = static.data(name="x", shape=[10, 10], dtype='float32')
                y = static.nn.fc(x, 10)
                z = static.nn.fc(y, 10)

                place = paddle.CPUPlace()
                exe = static.Executor(place)
                exe.run(static.default_startup_program())
                prog = static.default_main_program()

                path = "./temp/model.pdparams"
                paddle.save(prog.state_dict(), path)
                state_dict_load = paddle.load(path)
                prog.set_state_dict(state_dict_load)
        """

        if not isinstance(state_dict, dict):
            raise TypeError(
                "Type of `state_dict` should be dict, but received {}.".format(
                    type(state_dict)))

        vars_dict = {var.name: var for var in self.list_vars()}
        condition = True if 'StructuredToParameterName@@' in state_dict else False
        for name, value in state_dict.items():
            if condition:
                if name == "StructuredToParameterName@@":
                    continue
                if name in state_dict['StructuredToParameterName@@']:
                    name = state_dict['StructuredToParameterName@@'][name]
            if name in vars_dict:
                try:
                    vars_dict[name].set_value(value, scope)
                except ValueError as err:
                    warnings.warn(
                        ("Skip loading for '{}'. ".format(name) + str(err)))
                except TypeError as err:
                    warnings.warn(
                        ("Skip loading for '{}'. ".format(name) + str(err)))
            else:
6542 6543 6544
                warnings.warn(
                    ("Skip loading for '{0}'. Because '{0}' not in the program."
                     .format(name)))
6545

Y
Yu Yang 已提交
6546

6547
@six.add_metaclass(ParameterMetaClass)
Y
Yu Yang 已提交
6548
class Parameter(Variable):
6549
    """
6550
    Parameter is derived from Variable. A parameter is a persistable
6551
    Variable, and will be updated by optimizers after each iteration.
6552
    The training of a neural network is essentially the updating of
6553 6554
    its parameters.

6555
    Relative to a general Variable, a Parameter has several its own
6556 6557
    member variables:

6558 6559 6560 6561 6562 6563 6564 6565 6566 6567
    Args:
        trainable(bool): True if the parameter need to be updated after
            iterations.
        optimize_attr(map): Parameter attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the parameter. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this parameter.
6568
        need_clip (bool): Whether the parameter gradient need to be cliped
6569
            in optimizer. Default is True.
6570 6571
    """

6572 6573 6574 6575 6576 6577
    def __init__(self,
                 block,
                 shape,
                 dtype,
                 type=core.VarDesc.VarType.LOD_TENSOR,
                 **kwargs):
6578 6579 6580 6581 6582
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

Y
Yu Yang 已提交
6583
        if len(shape) == 0:
6584 6585
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")
Y
Yu Yang 已提交
6586 6587 6588

        for each in shape:
            if each < 0:
6589 6590 6591
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))
6592

6593 6594 6595 6596 6597 6598 6599
        Variable.__init__(self,
                          block,
                          persistable=True,
                          shape=shape,
                          dtype=dtype,
                          type=type,
                          **kwargs)
Y
Yu Yang 已提交
6600 6601 6602 6603
        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

6604 6605
        self.regularizer = kwargs.get('regularizer', None)

W
wanghaoshuang 已提交
6606
        self.do_model_average = kwargs.get('do_model_average', None)
W
wanghaoshuang 已提交
6607

6608 6609
        self.need_clip = kwargs.get('need_clip', True)

6610 6611
        self.is_distributed = False

6612 6613
        self.is_parameter = True

F
fengjiayi 已提交
6614
    def __str__(self):
6615
        return self._to_readable_code()
F
fengjiayi 已提交
6616

F
update  
fengjiayi 已提交
6617 6618 6619
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
D
dzhwinter 已提交
6620

F
update  
fengjiayi 已提交
6621 6622 6623 6624 6625 6626 6627 6628
        Args:
            throw_on_error(bool): raise exception when self is not initialized
                when throw_on_error is True
            with_details(bool): more details about variables and parameters
                (e.g. trainable, optimize_attr, ...) will be printed when with_details is True

        Returns(str): The debug string.

6629 6630 6631 6632 6633 6634 6635 6636 6637
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                rlt = fluid.layers.data("fake_data", shape=[1,1], dtype='float32')
                debug_str = prog.to_string(throw_on_error=True, with_details=False)
                print(debug_str)
F
update  
fengjiayi 已提交
6638
        """
6639 6640
        assert isinstance(throw_on_error, bool) and isinstance(
            with_details, bool)
F
update  
fengjiayi 已提交
6641 6642 6643
        if with_details:
            res_str = Variable.to_string(self, throw_on_error, True)
            additional_attr = ("trainable", "optimize_attr", "regularizer",
6644
                               "do_model_average", "need_clip")
F
update  
fengjiayi 已提交
6645
            for attr_name in additional_attr:
6646 6647
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))
F
update  
fengjiayi 已提交
6648 6649
        else:
            res_str = Variable.to_string(self, throw_on_error, False)
F
fengjiayi 已提交
6650 6651 6652 6653
        return res_str

    __repr__ = __str__

Y
Yu Yang 已提交
6654

6655 6656
class ParamBase(core.VarBase):
    """
6657 6658
    ParamBase is derived from Tensor( Which is the concept in Dygraph Mode).
    A ParamBase is a persistable Tensor, and will be updated by optimizers
6659
    after each iteration.
6660 6661 6662
    The training of a neural network is essentially the updating of
    its ParamBase.

6663
    Relative to a general Tensor, a ParamBase has several its own
6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675
    member variables:

    Args:
        trainable(bool): True if the ParamBase need to be updated after
            iterations.
        optimize_attr(map): ParamBase attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the ParamBase. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this ParamBase.
6676
        need_clip (bool): Whether the parameter gradient need to be cliped
6677
            in optimizer. Default is True.
6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702
    """

    @dygraph_only
    def __init__(self, shape, dtype, **kwargs):
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

        if len(shape) == 0:
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")

        for each in shape:
            if each < 0:
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))

        if dtype is not None:
            if not isinstance(dtype, core.VarDesc.VarType):
                dtype = convert_np_dtype_to_dtype_(dtype)

        name = kwargs.get('name', unique_name.generate('_param_base'))

6703 6704 6705 6706
        super(ParamBase,
              self).__init__(dtype if dtype else core.VarDesc.VarType.FP32,
                             list(shape) if shape else [], name,
                             core.VarDesc.VarType.LOD_TENSOR, True)
6707

6708 6709
        trainable = kwargs.get('trainable', True)
        self.stop_gradient = not trainable
6710 6711 6712 6713 6714 6715 6716

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

        self.regularizer = kwargs.get('regularizer', None)

        self.do_model_average = kwargs.get('do_model_average', None)

6717 6718
        self.need_clip = kwargs.get('need_clip', True)

6719
        self.is_distributed = kwargs.get('is_distributed', False)
6720
        # self.block = default_main_program().global_block()
6721

6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734
    @property
    def trainable(self):
        return not self.stop_gradient

    @trainable.setter
    def trainable(self, trainable):
        if isinstance(trainable, bool):
            self.stop_gradient = not trainable
        else:
            raise ValueError(
                "The type of trainable MUST be bool, but the type is ",
                type(trainable))

6735
    def __str__(self):
6736
        """
6737
        Convert a ParamBase object to a readable string.
6738

6739
        Returns(str): A readable string.
6740 6741 6742 6743

        Examples:
            .. code-block:: python

6744
                import paddle
6745 6746 6747 6748 6749 6750 6751
                linear = paddle.nn.Linear(3, 3)
                print(linear.weight)
                # Parameter containing:
                # Tensor(shape=[3, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=False,
                #        [[ 0.48948765,  0.05829060, -0.25524026],
                #         [-0.70368278,  0.52986908, -0.68742192],
                #         [-0.54217887,  0.48439729,  0.34082305]])
6752
        """
6753 6754
        return "Parameter containing:\n{tensor}".format(
            tensor=super(ParamBase, self).__str__())
6755

6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766
    def __deepcopy__(self, memo):
        """
        Deep copy parameter, it will always performs Tensor copy.

        Examples:
            .. code-block:: python

                import paddle
                import copy
                linear = paddle.nn.Linear(1, 3)
                linear_copy = copy.deepcopy(linear)
T
tangwei12 已提交
6767

6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785
                print(linear.weight)
                # Parameter containing:
                # Tensor(shape=[1, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
                #     [[-0.30929261, -0.90929240, -1.07851017]])

                print(linear_copy.weight)
                # Parameter containing:
                # Tensor(shape=[1, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
                #     [[-0.30929261, -0.90929240, -1.07851017]])

        """
        state = copy.deepcopy(self.__dict__, memo)
        state["name"] = self.name + unique_name.generate("_deepcopy")
        new_param = ParamBase(self.shape, self.dtype, **state)
        memo[id(self)] = new_param
        new_param.copy_(self, True)
        return new_param

6786 6787 6788 6789
    def _copy_to(self, device, blocking):
        state = copy.deepcopy(self.__dict__)
        new_param = ParamBase(self.shape, self.dtype, **state)
        core.varbase_copy(self, new_param, device, blocking)
6790 6791 6792 6793 6794 6795
        return new_param

    __repr__ = __str__


if hasattr(core, "eager"):
6796
    _core_eager_eagertensor = core.eager.Tensor
6797 6798 6799 6800 6801 6802
else:
    _core_eager_eagertensor = object


class EagerParamBase(_core_eager_eagertensor):
    """
6803 6804
    EagerParamBase is derived from Tensor( Which is the concept in Eager-Dygraph Mode).
    A EagerParamBase is a persistable Tensor, and will be updated by optimizers
6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821
    after each iteration.
    The training of a neural network is essentially the updating of
    its EagerParamBase.

    Relative to a general Tensor, a EagerParamBase has several its own
    member variables:

    Args:
        trainable(bool): True if the EagerParamBase need to be updated after
            iterations.
        optimize_attr(map): EagerParamBase attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the EagerParamBase. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this EagerParamBase.
6822
        need_clip (bool): Whether the parameter gradient need to be cliped
6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848
            in optimizer. Default is True.
    """

    @dygraph_only
    def __init__(self, shape, dtype, **kwargs):
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

        if len(shape) == 0:
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")

        for each in shape:
            if each < 0:
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))

        if dtype is not None:
            if not isinstance(dtype, core.VarDesc.VarType):
                dtype = convert_np_dtype_to_dtype_(dtype)

        name = kwargs.get('name', unique_name.generate('_eager_param_base'))

6849 6850 6851
        if isinstance(shape, core.eager.Tensor):
            shape = shape.numpy()

6852 6853 6854 6855
        super(EagerParamBase,
              self).__init__(dtype if dtype else core.VarDesc.VarType.FP32,
                             list(shape) if shape else [], name,
                             core.VarDesc.VarType.LOD_TENSOR, True)
6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869
        self.retain_grads()

        trainable = kwargs.get('trainable', True)
        self.stop_gradient = not trainable

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

        self.regularizer = kwargs.get('regularizer', None)

        self.do_model_average = kwargs.get('do_model_average', None)

        self.need_clip = kwargs.get('need_clip', True)

        self.is_distributed = kwargs.get('is_distributed', False)
6870 6871 6872
        # hook functions for lazy initialization
        self._init_func = None
        self._init_op_creator = None
6873 6874

    def set_init_func(self, obj):
6875
        self._init_func = obj
6876 6877 6878

    @dygraph_only
    def initialize(self):
6879 6880
        assert self._init_func is not None, "Required self._init_func is not None, but received None."
        self._init_func()
6881
        # clear function handle to release resource
6882
        self._init_func = None
6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896

    @property
    def trainable(self):
        return not self.stop_gradient

    @trainable.setter
    def trainable(self, trainable):
        if isinstance(trainable, bool):
            self.stop_gradient = not trainable
        else:
            raise ValueError(
                "The type of trainable MUST be bool, but the type is ",
                type(trainable))

6897 6898 6899 6900 6901 6902 6903
    def _create_init_op(self, block):
        """
        Call init_op_creator function to create initializer operation in block.
        """
        assert self._init_op_creator is not None, "Required self._init_op_creator is not None, but received None."
        self._init_op_creator(block)

6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958
    def __str__(self):
        """
        Convert a EagerParamBase object to a readable string.

        Returns(str): A readable string.

        Examples:
            .. code-block:: python

                import paddle
                linear = paddle.nn.Linear(3, 3)
                print(linear.weight)
                # Parameter containing:
                # Tensor(shape=[3, 3], dtype=float32, place=CUDAPlace(0), stop_gradient=False,
                #        [[ 0.48948765,  0.05829060, -0.25524026],
                #         [-0.70368278,  0.52986908, -0.68742192],
                #         [-0.54217887,  0.48439729,  0.34082305]])
        """
        return "Parameter containing:\n{tensor}".format(
            tensor=super(EagerParamBase, self).__str__())

    def __deepcopy__(self, memo):
        """
        Deep copy parameter, it will always performs Tensor copy.

        Examples:
            .. code-block:: python

                import paddle
                import copy
                linear = paddle.nn.Linear(1, 3)
                linear_copy = copy.deepcopy(linear)

                print(linear.weight)
                # Parameter containing:
                # Tensor(shape=[1, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
                #     [[-0.30929261, -0.90929240, -1.07851017]])

                print(linear_copy.weight)
                # Parameter containing:
                # Tensor(shape=[1, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
                #     [[-0.30929261, -0.90929240, -1.07851017]])

        """
        state = copy.deepcopy(self.__dict__, memo)
        state["name"] = self.name + unique_name.generate("_deepcopy")
        new_param = EagerParamBase(self.shape, self.dtype, **state)
        memo[id(self)] = new_param
        new_param.copy_(self, True)
        return new_param

    def _copy_to(self, device, blocking):
        state = copy.deepcopy(self.__dict__)
        new_param = EagerParamBase(self.shape, self.dtype, **state)
        core.eager.tensor_copy(self, new_param, device, blocking)
6959 6960
        return new_param

6961 6962 6963
    __repr__ = __str__


Y
Yu Yang 已提交
6964
# program is a global instance.
Y
Yu Yang 已提交
6965 6966
_main_program_ = Program()
_startup_program_ = Program()
6967
_startup_program_._is_start_up_program_ = True
6968

6969

6970
def default_startup_program():
Y
Yu Yang 已提交
6971
    """
Y
yuyang18 已提交
6972 6973
    Get default/global startup program.

6974
    The :code:`paddle.nn` function will append the initialization operators into startup program.
6975
    The :code:`startup_program` will initialize the parameters by the OPs.
T
tangwei12 已提交
6976

6977 6978
    This method will return the default or the current startup program. Users can use
    :ref:`api_paddle_fluid_framework_program_guard`  to switch :ref:`api_paddle_fluid_framework_Program` .
Y
yuyang18 已提交
6979

6980 6981
    Returns:
        Program: current default startup program.
6982

6983
    Returns type:
6984 6985 6986 6987

    Examples:
        .. code-block:: python

6988
            import paddle
6989

6990
            paddle.enable_static()
6991 6992 6993 6994
            x = paddle.static.data(name="x", shape=[-1, 784], dtype='float32')
            out = paddle.static.nn.fc(name="fc", x=x, size=10, activation="relu")
            print("main program is: {}".format(paddle.static.default_main_program()))
            print("start up program is: {}".format(paddle.static.default_startup_program()))
Y
Yu Yang 已提交
6995
    """
Y
Yu Yang 已提交
6996
    return _startup_program_
6997

6998

6999
def default_main_program():
Y
Yu Yang 已提交
7000
    """
7001
    This API can be used to get ``default main program`` which store the
7002
    descriptions of Ops and tensors.
T
tangwei12 已提交
7003

7004 7005
    For example ``z = paddle.add(x, y)`` will create a new ``add``
    Op and a new ``z`` tensor, and they will be recorded in ``default main program`` .
Y
yuyang18 已提交
7006

7007
    The ``default main program`` is the default value for ``Program`` parameter in
7008
    a lot of APIs. For example, the :code:`Executor.run()` will execute the
Y
yuyang18 已提交
7009
    :code:`default_main_program` when the program is not specified.
7010

7011
    If you want to switch the ``default main program``, you can use :ref:`api_paddle_fluid_framework_program_guard` .
T
tangwei12 已提交
7012

Y
Yu Yang 已提交
7013
    Returns:
7014
        Program: A ``Program`` which holding the descriptions of OPs and tensors in the network.
7015 7016 7017 7018

    Examples:
        ..  code-block:: python

7019
            import paddle
7020

7021
            paddle.enable_static()
7022
            # Sample Network:
7023 7024 7025
            x = paddle.static.data(name='x', shape=[100, 100], dtype='float32')
            y = paddle.static.data(name='x', shape=[100, 100], dtype='float32')
            out = paddle.add(x, y)
7026

7027 7028 7029
            #print the number of blocks in the program, 1 in this case
            print(paddle.static.default_main_program().num_blocks) # 1
            #print the default_main_program
7030
            print(paddle.static.default_main_program())
Y
Yu Yang 已提交
7031
    """
Y
Yu Yang 已提交
7032
    return _main_program_
Y
Yu Yang 已提交
7033 7034 7035 7036 7037


def switch_main_program(program):
    """
    Switch the main program to a new program.
7038

Y
Yu Yang 已提交
7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052
    Args:
        program(Program): The new main program

    Returns:
        Program: The previous main program
    """
    global _main_program_
    prev_program = _main_program_
    _main_program_ = program
    return prev_program


def switch_startup_program(program):
    """
7053
    Switch the startup program to a new program
Y
Yu Yang 已提交
7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065
    Args:
        program(Program): The new startup program

    Returns:
        Program: The previous startup program
    """
    global _startup_program_
    prev_program = _startup_program_
    _startup_program_ = program
    return prev_program


S
rename  
sneaxiy 已提交
7066
@signature_safe_contextmanager
Y
Yu Yang 已提交
7067 7068
def program_guard(main_program, startup_program=None):
    """
7069 7070
    :api_attr: Static Graph

7071 7072 7073
    Change the global main program and startup program with ``with`` statement.
    Layer functions in the Python ``with`` block will append operators and
    Tensors to the new main programs.
7074

G
guofei 已提交
7075
    Args:
7076
        main_program(Program): New main program inside ``with`` statement.
7077 7078
        startup_program(Program, optional): New startup program inside ``with``
            statement. :code:`None` means not changing startup program,
G
guofei 已提交
7079 7080 7081
            default_startup_program is still used.
            Default: None.

Y
Yu Yang 已提交
7082
    Examples:
7083
       .. code-block:: python
T
tangwei12 已提交
7084

7085
          import paddle
Y
yuyang18 已提交
7086

7087 7088 7089 7090 7091
          paddle.enable_static()
          main_program = paddle.static.Program()
          startup_program = paddle.static.Program()
          with paddle.static.program_guard(main_program, startup_program):
              data = paddle.static.data(name='image', shape=[None, 784, 784], dtype='float32')
7092
              hidden = paddle.static.nn.fc(x=data, size=10, activation='relu')
Y
yuyang18 已提交
7093 7094 7095

    Notes: The temporary :code:`Program` can be used if the user does not need
    to construct either of startup program or main program.
7096

Y
Yu Yang 已提交
7097
    Examples:
7098
       .. code-block:: python
Y
yuyang18 已提交
7099

7100
          import paddle
7101

7102 7103 7104 7105 7106
          paddle.enable_static()
          main_program = paddle.static.Program()
          # does not care about startup program. Just pass a temporary value.
          with paddle.static.program_guard(main_program, paddle.static.Program()):
              data = paddle.static.data(name='image', shape=[None, 784, 784], dtype='float32')
T
tangwei12 已提交
7107

Y
Yu Yang 已提交
7108
    """
7109
    from .data_feeder import check_type
7110 7111
    check_type(main_program, 'main_program', Program,
               'paddle.static.program_guard')
Y
Yu Yang 已提交
7112 7113
    main_program = switch_main_program(main_program)
    if startup_program is not None:
7114
        check_type(startup_program, 'startup_program', Program,
7115
                   'paddle.static.program_guard')
7116 7117
        # Tag the program __is_start_up as True
        startup_program._is_start_up_program_ = True
Y
Yu Yang 已提交
7118
        startup_program = switch_startup_program(startup_program)
7119 7120 7121 7122 7123 7124
    try:
        yield
    finally:
        switch_main_program(main_program)
        if startup_program is not None:
            switch_startup_program(startup_program)
X
xuwei06 已提交
7125 7126


W
Wu Yi 已提交
7127
def _get_var(name, program=None):
X
xuwei06 已提交
7128
    """
Y
yuyang18 已提交
7129
    Get a variable by name from the global block of a program.
F
fengjiayi 已提交
7130

X
xuwei06 已提交
7131 7132 7133
    Args:
        name(str): name of the variable
        program(Program|None): program object.
T
tangwei12 已提交
7134
        If None, default_global_program() will be used.
X
xuwei06 已提交
7135 7136 7137 7138 7139 7140 7141

    Returns:
        Variable
    """
    if program is None:
        program = default_main_program()
    assert isinstance(name, str)
7142
    assert isinstance(program, Program)
X
xuwei06 已提交
7143 7144

    return program.global_block().var(name)
7145 7146


S
rename  
sneaxiy 已提交
7147
@signature_safe_contextmanager
L
lujun 已提交
7148 7149
def _dygraph_guard(tracer):
    global _dygraph_tracer_
7150
    tmp_tracer = _dygraph_tracer_
L
lujun 已提交
7151
    _dygraph_tracer_ = tracer
7152
    core._switch_tracer(tracer)
M
minqiyang 已提交
7153

7154 7155 7156
    try:
        yield
    finally:
7157 7158
        core._switch_tracer(tmp_tracer)
        _dygraph_tracer_ = tmp_tracer
P
Paddle CI 已提交
7159 7160


S
rename  
sneaxiy 已提交
7161
@signature_safe_contextmanager
L
lujun 已提交
7162
def _dygraph_place_guard(place):
7163 7164 7165
    global _global_expected_place_
    tmp_place = _global_expected_place_
    _global_expected_place_ = place
7166 7167
    _set_dygraph_tracer_expected_place(place)

7168 7169 7170
    try:
        yield
    finally:
7171
        _global_expected_place_ = tmp_place
J
Jiabin Yang 已提交
7172
        _set_dygraph_tracer_expected_place(_global_expected_place_)
7173 7174


7175 7176 7177 7178 7179 7180 7181 7182 7183 7184
def switch_device(device):
    global _current_device
    pre_device = _current_device
    _current_device = device
    return pre_device


@signature_safe_contextmanager
def device_guard(device=None):
    """
7185

7186 7187
    Note:
        The API only supports static mode.
7188 7189 7190 7191

    A context manager that specifies the device on which the OP will be placed.

    Args:
7192
        device(str|None): Specify the device to use in the context. It should be ``cpu``,
7193
            ``gpu`` or ``gpu:x``, where ``x`` is the index of the GPUs.
7194 7195 7196 7197 7198 7199 7200
            When it is set to 'cpu' or 'gpu', all OPs created in the context will be
            placed on CPUPlace or CUDAPlace. When 'gpu' is set and the program runs on
            single-card, the device index will be the same as the device on which the
            executor runs. Default: None, OPs in this context will be automatically
            assigned devices.

    Examples:
7201

7202
        .. code-block:: python
7203

7204
            # required: gpu
Z
Zhang Ting 已提交
7205
            import paddle
7206

Z
Zhang Ting 已提交
7207 7208 7209
            paddle.enable_static()
            support_gpu = paddle.is_compiled_with_cuda()
            place = paddle.CPUPlace()
7210
            if support_gpu:
Z
Zhang Ting 已提交
7211
                place = paddle.CUDAPlace(0)
7212 7213

            # if GPU is supported, the three OPs below will be automatically assigned to CUDAPlace(0)
Z
Zhang Ting 已提交
7214 7215 7216
            data1 = paddle.full(shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32')
            data2 = paddle.full(shape=[1, 3, 64], fill_value=0.5, dtype='float32')
            shape = paddle.shape(data2)
7217

Z
Zhang Ting 已提交
7218
            with paddle.static.device_guard("cpu"):
7219
                # Ops created here will be placed on CPUPlace
Z
Zhang Ting 已提交
7220 7221
                shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4])
            with paddle.static.device_guard('gpu'):
7222
                # if GPU is supported, OPs created here will be placed on CUDAPlace(0), otherwise on CPUPlace
Z
Zhang Ting 已提交
7223
                out = paddle.reshape(data1, shape=shape)
7224

Z
Zhang Ting 已提交
7225 7226
            exe = paddle.static.Executor(place)
            exe.run(paddle.static.default_startup_program())
7227 7228 7229
            result = exe.run(fetch_list=[out])
    """

7230 7231 7232 7233 7234
    index = None
    if device and ':' in device:
        device, index = device.split(':')
        if device == 'cpu':
            raise ValueError("Should not set device id for cpu.")
7235
    if device not in ['cpu', 'gpu', 'npu', 'xpu', 'mlu', '', None]:
7236
        raise ValueError(
7237
            "The Attr(device) should be 'cpu' 'npu' 'xpu' 'mlu' or 'gpu', and it can also be empty string or None "
7238
            "when there is no need to specify device. But received %s" % device)
7239 7240
    if index:
        device = ":".join([device, index])
7241
    pre_device = switch_device(device)
7242 7243 7244 7245
    try:
        yield
    finally:
        switch_device(pre_device)
G
guofei 已提交
7246 7247


7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278
def _switch_cuda_graph_mode(cuda_graph_attr):
    global _current_cuda_graph_mode
    pre_mode = _current_cuda_graph_mode
    _current_cuda_graph_mode = cuda_graph_attr
    return pre_mode


@signature_safe_contextmanager
def _cuda_graph_guard(cuda_graph_attr=None):
    """

    Note:
        The API only supports static mode.

    A context manager that specifies the cuda_graph_mode which indicating the cuda graph capture under static mode.

    Args:
        cuda_graph_attr(str|None): The cuda graph attr with the format of:
                                   cuda_graph_capture_mode;memory_pool_id;cuda_graph_id
    """
    assert not _non_static_mode(
    ), "cuda_graph_guard only works under static mode"
    assert core.is_compiled_with_cuda(
    ), "cuda_graph_guard context can be only used when Paddle is compiled with cuda"
    pre_mode = _switch_cuda_graph_mode(cuda_graph_attr)
    try:
        yield
    finally:
        _switch_cuda_graph_mode(pre_mode)


G
guofei 已提交
7279 7280 7281
def set_flags(flags):
    """
    This function sets the GFlags value in Paddle.
7282
    For FLAGS please refer to :ref:`en_guides_flags_flags`
G
guofei 已提交
7283 7284 7285 7286 7287 7288 7289

    Args:
        flags (dict): A dict contains flags and its value.

    Examples:
            .. code-block:: python

7290 7291
                import paddle
                paddle.set_flags({'FLAGS_eager_delete_tensor_gb': 1.0})
G
guofei 已提交
7292 7293 7294 7295
    """
    if not isinstance(flags, dict):
        raise TypeError('flags in set_flags should be a dict')
    for key, value in flags.items():
7296 7297
        if _global_flags().is_public(key):
            _global_flags()[key] = value
G
guofei 已提交
7298 7299 7300 7301 7302 7303 7304 7305
        else:
            raise ValueError(
                "Flag %s cannot set its value through this function." % (key))


def get_flags(flags):
    """
    This function gets the GFlags value in Paddle.
7306
    For FLAGS please refer to :ref:`en_guides_flags_flags`
G
guofei 已提交
7307 7308 7309 7310 7311 7312 7313 7314 7315 7316

    Args:
        flags(list|tuple|str): A list/tuple of string or a string which is the flag's name.

    Returns:
        flag's value in Paddle.

    Examples:
        .. code-block:: python

7317
            import paddle
G
guofei 已提交
7318 7319

            flags = ['FLAGS_eager_delete_tensor_gb', 'FLAGS_check_nan_inf']
7320
            res = paddle.get_flags(flags)
G
guofei 已提交
7321 7322 7323 7324 7325 7326
            print(res)
            # {'FLAGS_eager_delete_tensor_gb': 0.0, 'FLAGS_check_nan_inf': False}
    """
    flags_value = {}
    if isinstance(flags, (list, tuple)):
        for key in flags:
7327 7328
            if (_global_flags().is_public(key)):
                value = _global_flags()[key]
G
guofei 已提交
7329 7330 7331 7332 7333 7334 7335
                temp = {key: value}
                flags_value.update(temp)
            else:
                raise ValueError(
                    'Flag %s cannot get its value through this function.' %
                    (key))
    elif isinstance(flags, str):
7336 7337
        if (_global_flags().is_public(flags)):
            value = _global_flags()[flags]
G
guofei 已提交
7338 7339 7340 7341 7342 7343 7344 7345
            temp = {flags: value}
            flags_value.update(temp)
        else:
            raise ValueError(
                'Flag %s cannot get its value through this function.' % (flags))
    else:
        raise TypeError('Flags in get_flags should be a list, tuple or string.')
    return flags_value
7346 7347 7348 7349 7350 7351 7352


def _get_paddle_place(place):
    "convert the string to paddle Place"
    if place is None:
        return place
    if isinstance(place, (core.Place, core.XPUPlace, core.CPUPlace,
7353
                          core.CUDAPinnedPlace, core.CUDAPlace, core.NPUPlace,
7354
                          core.IPUPlace, core.MLUPlace, core.CustomPlace)):
7355 7356 7357 7358 7359 7360 7361 7362 7363
        return place

    if not isinstance(place, str):
        raise ValueError(
            "place only support string which is 'Place' and so on.")

    place = place.lower()
    if (place == "cpu"):
        return core.CPUPlace()
7364

7365 7366 7367
    if (place == "device"):
        return core.Place()

7368
    # GPU
7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383
    avaliable_gpu_place = re.match(r'gpu:\d+', place)
    if place == "gpu_pinned" or place == "gpu" or avaliable_gpu_place:
        if not core.is_compiled_with_cuda():
            raise ValueError(
                "The device should not be {}, since PaddlePaddle is " \
                "not compiled with CUDA".format(avaliable_gpu_place))
        if place == "gpu_pinned":
            return core.CUDAPinnedPlace()
        elif place == "gpu":
            return core.CUDAPlace(0)
        else:
            place_info_list = place.split(':', 1)
            device_id = place_info_list[1]
            device_id = int(device_id)
            return core.CUDAPlace(device_id)
7384 7385

    # XPU
7386 7387 7388 7389 7390 7391 7392 7393 7394 7395
    avaliable_xpu_place = re.match(r'xpu:\d+', place)
    if avaliable_xpu_place:
        if not core.is_compiled_with_xpu():
            raise ValueError(
                "The device should not be {}, since PaddlePaddle is " \
                "not compiled with XPU".format(avaliable_xpu_place))
        place_info_list = place.split(':', 1)
        device_id = place_info_list[1]
        device_id = int(device_id)
        return core.XPUPlace(device_id)
7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408

    # NPU
    avaliable_npu_place = re.match(r'npu:\d+', place)
    if avaliable_npu_place:
        if not core.is_compiled_with_npu():
            raise ValueError(
                "The device should not be {}, since PaddlePaddle is " \
                "not compiled with NPU".format(avaliable_npu_place))
        place_info_list = place.split(':', 1)
        device_id = place_info_list[1]
        device_id = int(device_id)
        return core.NPUPlace(device_id)

J
jianghaicheng 已提交
7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420
    # IPU
    avaliable_ipu_place = re.match(r'ipu:\d+', place)
    if avaliable_ipu_place:
        if not core.is_compiled_with_ipu():
            raise ValueError(
                "The device should not be {}, since PaddlePaddle is " \
                "not compiled with IPU".format(avaliable_ipu_place))
        place_info_list = place.split(':', 1)
        device_id = place_info_list[1]
        device_id = int(device_id)
        return core.IPUPlace(device_id)

7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432
    # MLU
    avaliable_mlu_place = re.match(r'mlu:\d+', place)
    if avaliable_mlu_place:
        if not core.is_compiled_with_mlu():
            raise ValueError(
                "The device should not be {}, since PaddlePaddle is " \
                "not compiled with MLU".format(avaliable_mlu_place))
        place_info_list = place.split(':', 1)
        device_id = place_info_list[1]
        device_id = int(device_id)
        return core.MLUPlace(device_id)

7433
    raise ValueError(
7434 7435
        "Paddle supports CPUPlace, CUDAPlace,CUDAPinnedPlace, XPUPlace, IPUPlace, MLUPlace and NPUPlace, but received {}."
        .format(place))
7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448


def _get_paddle_place_list(places):

    if not isinstance(places, (list, tuple)):
        raise TypeError("places must to be List or Tuple")

    ret = []
    for p in places:
        p = _get_paddle_place(p)
        ret.append(p)

    return ret