framework.py 162.4 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

Y
Yu Yang 已提交
17
import collections
X
Xin Pan 已提交
18
from collections import defaultdict
W
WangZhen 已提交
19
from collections import Iterable
Q
qiaolongfei 已提交
20
import contextlib
21
from .wrapped_decorator import signature_safe_contextmanager, wrap_decorator
P
peizhilin 已提交
22
import os
F
fengjiayi 已提交
23
import re
24
import traceback
25
import six
26

Y
Yu Yang 已提交
27
import numpy as np
28
import subprocess
S
sneaxiy 已提交
29
import multiprocessing
30
import sys
31
import logging
M
minqiyang 已提交
32
from .. import compat as cpt
33
from .proto import framework_pb2
34 35

from . import core
36
from . import unique_name
37 38
import paddle.version as fluid_version
import warnings
Y
Yu Yang 已提交
39

40
__all__ = [
41 42 43 44
    'Program',
    'default_startup_program',
    'default_main_program',
    'program_guard',
45
    'name_scope',
S
sneaxiy 已提交
46 47 48
    'cuda_places',
    'cpu_places',
    'cuda_pinned_places',
L
lujun 已提交
49
    'in_dygraph_mode',
C
chengduo 已提交
50
    'is_compiled_with_cuda',
51
    'Variable',
52
    'load_op_library',
53
    'require_version',
54
]
Y
Yu Yang 已提交
55

Q
qiaolongfei 已提交
56 57 58 59
EMPTY_VAR_NAME = core.kEmptyVarName()
TEMP_VAR_NAME = core.kTempVarName()
GRAD_VAR_SUFFIX = core.kGradVarSuffix()
ZERO_VAR_SUFFIX = core.kZeroVarSuffix()
W
Wu Yi 已提交
60 61
CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName()

L
lujun 已提交
62 63
_dygraph_tracer_ = None
_dygraph_current_expected_place_ = None
64 65


66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
def require_version(min_version, max_version=None):
    """
        Check if the installed version of PaddlePaddle is in [min_version, max_version],
        if the installed version is lower than ``min_version`` or higher than ``max_version``,
        an exception will be thrown, NO returns if the installed version is satisfied.

        Args:
            min_version (str): the minimum version required (like '1.4.0').
            max_version (str, optional): the max version required (like '1.6.0'), default is None,
                meaning any version equal or higher than ``min_version`` is acceptable.

        Returns:
            None.

        Raises:
            TypeError: if the type of ``min_version`` is not str.
            TypeError: if the type of ``max_version`` is not str or type(None).
            ValueError: if the value of ``min_version`` is not in version format.
            ValueError: if the value of ``max_version`` is not in version format or None.
            Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                # any version >= 0.1.0 is acceptable.
                fluid.require_version('0.1.0')

                # if 0.1.0 <= version <= 10.0.0, it is acceptable.
                fluid.require_version(min_version='0.1.0', max_version='10.0.0')
        """
    if not isinstance(min_version, str):
        raise TypeError(
            "The type of 'min_version' in require_version must be str, but received %s."
            % (type(min_version)))

    if not isinstance(max_version, (str, type(None))):
        raise TypeError(
            "The type of 'max_version' in require_version must be str or type(None), but received %s."
            % (type(max_version)))

    check_format = re.match(r'\d+(\.\d+){0,3}', min_version)
    if check_format is None or check_format.group() != min_version:
        raise ValueError(
            "The value of 'min_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
            "like '1.5.2.0', but received %s" % min_version)

    if max_version is not None:
        check_format = re.match(r'\d+(\.\d+){0,3}', max_version)
        if check_format is None or check_format.group() != max_version:
            raise ValueError(
                "The value of 'max_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
                "like '1.5.2.0', but received %s" % max_version)

    version_installed = [
        fluid_version.major, fluid_version.minor, fluid_version.patch,
        fluid_version.rc
    ]
    zero_version = ['0', '0', '0', '0']

    def version_cmp(ver_a, ver_b):
        for i in six.moves.range(len(ver_a)):
            if int(ver_a[i]) > int(ver_b[i]):
                return 1
            elif int(ver_a[i]) < int(ver_b[i]):
                return -1
        return 0

    if version_cmp(version_installed, zero_version) == 0:
        if max_version is not None:
            warnings.warn(
                "PaddlePaddle version in [%s, %s] required, but %s installed. "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, max_version, fluid_version.full_version))
        else:
            warnings.warn(
                "PaddlePaddle version %s or higher is required, but %s installed, "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, fluid_version.full_version))
        return

    min_version_split = min_version.split('.')
    min_version_to_check = min_version_split + zero_version[len(
        min_version_split):]

    if max_version is not None:
        max_version_split = max_version.split('.')
        max_version_to_check = max_version_split + zero_version[len(
            max_version_split):]

        if version_cmp(version_installed,
                       max_version_to_check) > 0 or version_cmp(
                           version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version in [%s, %s] required, but %s installed."
                % (min_version, max_version, fluid_version.full_version))
    else:
        if version_cmp(version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version %s or higher is required, but %s installed, "
                "please upgrade your PaddlePaddle to %s or other higher version."
                % (min_version, fluid_version.full_version, min_version))


L
lujun 已提交
173
def in_dygraph_mode():
L
lujun 已提交
174
    """
Y
Youwei Song 已提交
175 176
    This function checks whether the program runs in dynamic graph mode or not.
    You can turn on dynamic graph mode with :ref:`api_fluid_dygraph_guard` api.
L
lujun 已提交
177 178

    Returns:
Y
Youwei Song 已提交
179
        bool: Whether the program is running in dynamic graph mode.
L
lujun 已提交
180 181 182 183

    Examples:
        .. code-block:: python

184
            import paddle.fluid as fluid
L
lujun 已提交
185
            if fluid.in_dygraph_mode():
Y
Youwei Song 已提交
186 187 188
                print('running in dygraph mode')
            else:
                print('not running in dygraph mode')
L
lujun 已提交
189 190

    """
L
lujun 已提交
191
    return _dygraph_tracer_ is not None
192 193


194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
def _dygraph_not_support_(func):
    def __impl__(*args, **kwargs):
        assert not in_dygraph_mode(
        ), "We don't support %s in Dygraph mode" % func.__name__
        return func(*args, **kwargs)

    return __impl__


def _dygraph_only_(func):
    def __impl__(*args, **kwargs):
        assert in_dygraph_mode(
        ), "We Only support %s in Dygraph mode, please use fluid.dygraph.guard() as context to run it in Dygraph Mode" % func.__name__
        return func(*args, **kwargs)

    return __impl__


dygraph_not_support = wrap_decorator(_dygraph_not_support_)
dygraph_only = wrap_decorator(_dygraph_only_)


L
lujun 已提交
216 217
def _dygraph_tracer():
    return _dygraph_tracer_
218

W
Wu Yi 已提交
219

M
minqiyang 已提交
220
def _current_expected_place():
L
lujun 已提交
221
    return _dygraph_current_expected_place_
M
minqiyang 已提交
222 223


S
sneaxiy 已提交
224
def _cpu_num():
225
    if "CPU_NUM" not in os.environ.keys():
C
chengduo 已提交
226 227 228 229 230 231 232 233
        if multiprocessing.cpu_count() > 1:
            sys.stderr.write(
                '!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.\n'
                'CPU_NUM indicates that how many CPUPlace are used in the current task.\n'
                'And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.\n\n'
                'export CPU_NUM={} # for example, set CPU_NUM as number of physical CPU core which is {}.\n\n'
                '!!! The default number of CPU_NUM=1.\n'.format(
                    multiprocessing.cpu_count(), multiprocessing.cpu_count()))
C
chengduo 已提交
234
        os.environ['CPU_NUM'] = str(1)
235
    cpu_num = os.environ.get('CPU_NUM')
C
chengduo 已提交
236 237 238 239 240 241 242 243 244 245
    return int(cpu_num)


def _cuda_ids():
    gpus_env = os.getenv("FLAGS_selected_gpus")
    if gpus_env:
        device_ids = [int(s) for s in gpus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_cuda_device_count())
    return device_ids
S
sneaxiy 已提交
246 247


C
chengduo 已提交
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
def is_compiled_with_cuda():
    """
    Whether this whl package can be used to run the model on GPU.

    Returns (bool): support gpu or not.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            support_gpu = fluid.is_compiled_with_cuda()
    """
    return core.is_compiled_with_cuda()


H
hong 已提交
263 264 265 266 267 268 269 270 271 272 273 274 275
def _var_base_to_np(var_base):
    """
    convert VarBase tp numpy
    
    Args:
        var_base(VarBase) : the VarBase to convert
    Returns (np.ndarray): the np.ndarray contain the value of VarBase

    """
    var = var_base._copy_to(core.CPUPlace(), True)
    return np.array(var.value().get_tensor())


S
sneaxiy 已提交
276
def cuda_places(device_ids=None):
L
lujun 已提交
277
    """
278 279 280 281 282
    **Note**:
        For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device.
        The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable.

    This function creates a list of :code:`fluid.CUDAPlace` objects.
S
add doc  
sneaxiy 已提交
283 284

    If :code:`device_ids` is None, environment variable of
285
    :code:`FLAGS_selected_gpus` would be checked first. For example, if
S
add doc  
sneaxiy 已提交
286 287 288
    :code:`FLAGS_selected_gpus=0,1,2`, the returned list would
    be [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
    If :code:`FLAGS_selected_gpus` is not set, all visible
289
    gpu places would be returned according to the :code:`CUDA_VISIBLE_DEVICES` environment variable.
S
add doc  
sneaxiy 已提交
290 291

    If :code:`device_ids` is not None, it should be the device
292
    ids of GPUs. For example, if :code:`device_ids=[0,1,2]`,
S
add doc  
sneaxiy 已提交
293 294 295
    the returned list would be 
    [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
    
296 297
    Parameters:
        device_ids (list or tuple of int, optional): list of GPU device ids.
S
add doc  
sneaxiy 已提交
298 299

    Returns:
300
        list of fluid.CUDAPlace: Created GPU place list.
L
lujun 已提交
301 302 303 304

    Examples:
        .. code-block:: python

305
            import paddle.fluid as fluid
L
lujun 已提交
306 307 308
            cuda_places = fluid.cuda_places()

    """
S
sneaxiy 已提交
309 310 311
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_ids is None:
C
chengduo 已提交
312
        device_ids = _cuda_ids()
S
sneaxiy 已提交
313 314 315 316 317 318
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.CUDAPlace(dev_id) for dev_id in device_ids]


def cpu_places(device_count=None):
L
lujun 已提交
319
    """
320
    This function creates a list of :code:`fluid.CPUPlace` objects, and returns the created list.
S
add doc  
sneaxiy 已提交
321 322 323
    
    If :code:`device_count` is None, the device count would
    be determined by environment variable :code:`CPU_NUM`. 
C
chengduo 已提交
324 325
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
326 327
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
328

329 330
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
331 332

    Returns:
333
        list of fluid.CPUPlace: Created list of CPU places.
L
lujun 已提交
334 335 336 337

    Examples:
        .. code-block:: python

338
            import paddle.fluid as fluid
L
lujun 已提交
339 340 341
            cpu_places = fluid.cpu_places()
    """

S
sneaxiy 已提交
342 343 344 345 346 347
    if device_count is None:
        device_count = _cpu_num()
    return [core.CPUPlace()] * device_count


def cuda_pinned_places(device_count=None):
L
lujun 已提交
348
    """
349
    This function creates a list of :code:`fluid.CUDAPinnedPlace` objects.
S
add doc  
sneaxiy 已提交
350 351 352

    If :code:`device_count` is None, the device count would
    be determined by environment variable :code:`CPU_NUM`. 
353 354 355 356
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
357

358 359
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
360 361

    Returns:
362
        list of fluid.CUDAPinnedPlace: Created list of CUDA pinned places.
L
lujun 已提交
363 364 365 366

    Examples:
        .. code-block:: python

367
            import paddle.fluid as fluid
L
lujun 已提交
368 369 370 371 372
            cuda_pinned_places_cpu_num = fluid.cuda_pinned_places()
            # or
            cuda_pinned_places = fluid.cuda_pinned_places(1)

    """
S
sneaxiy 已提交
373 374 375 376 377 378 379
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_count is None:
        device_count = _cpu_num()
    return [core.cuda_pinned_places()] * device_count


380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
class NameScope(object):
    def __init__(self, name="", parent=None):
        self._children = dict()
        self._name = name
        self._parent = parent

    def child(self, prefix):
        if prefix not in self._children:
            new_child = NameScope(prefix, self)
            self._children[prefix] = [new_child]
        else:
            new_child = NameScope(prefix + "_%d" % len(self._children[prefix]),
                                  self)
            self._children[prefix].append(new_child)
        return new_child

    def parent(self):
        return self._parent

    def name(self):
        return self._name


_name_scope = NameScope()


S
rename  
sneaxiy 已提交
406
@signature_safe_contextmanager
407 408 409 410
def name_scope(prefix=None):
    """
    Generate hierarchical name prefix for the operators.

T
Tao Luo 已提交
411 412 413
    Note: 
        This should only used for debugging and visualization purpose.
        Don't use it for serious analysis such as graph/program transformations.
414 415

    Args:
T
Tao Luo 已提交
416
        prefix(str, optional): prefix. Default is none.
417 418 419

    Examples:
        .. code-block:: python
T
Tink_Y 已提交
420

421
          import paddle.fluid as fluid
422
          with fluid.name_scope("s1"):
T
Tao Luo 已提交
423 424 425 426 427 428
             a = fluid.data(name='data', shape=[None, 1], dtype='int32')
             b = a + 1
             with fluid.name_scope("s2"):
                c = b * 1
             with fluid.name_scope("s3"):
                d = c / 1
429
          with fluid.name_scope("s1"):
T
Tao Luo 已提交
430
                f = fluid.layers.pow(d, 2.0)
431
          with fluid.name_scope("s4"):
T
Tao Luo 已提交
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
                g = f - 1

          # Op are created in the default main program.  
          for op in fluid.default_main_program().block(0).ops:
              # elementwise_add is created in /s1/
              if op.type == 'elementwise_add':
                  assert op.desc.attr("op_namescope") == '/s1/'
              # elementwise_mul is created in '/s1/s2'
              elif op.type == 'elementwise_mul':
                  assert op.desc.attr("op_namescope") == '/s1/s2/'
              # elementwise_div is created in '/s1/s3'
              elif op.type == 'elementwise_div':
                  assert op.desc.attr("op_namescope") == '/s1/s3/'
              # elementwise_sum is created in '/s4'
              elif op.type == 'elementwise_sub':
                  assert op.desc.attr("op_namescope") == '/s4/'
              # pow is created in /s1_1/
              elif op.type == 'pow':
                  assert op.desc.attr("op_namescope") == '/s1_1/'
451 452
    """
    # TODO(panyx0718): Only [0-9a-z].
453 454 455 456 457 458 459 460 461
    # in dygraph we don't need namescope since it will cause mem leak
    if not in_dygraph_mode():
        assert prefix, "namescope prefix cannot be empty."
        global _name_scope
        _name_scope = _name_scope.child(prefix)
        yield
        _name_scope = _name_scope.parent()
    else:
        yield
462 463 464 465 466 467 468 469 470 471 472 473


def _full_name_scope():
    global _name_scope
    scope = _name_scope
    name = ""
    while scope:
        name = scope.name() + "/" + name
        scope = scope.parent()
    return name


W
Wu Yi 已提交
474 475 476
def generate_control_dev_var_name():
    import random
    return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random())
Q
qiaolongfei 已提交
477 478 479 480


def grad_var_name(var_name):
    """
481 482
    Returns:
        str: gradient name for a certain var name
Q
qiaolongfei 已提交
483 484 485
    """
    return var_name + GRAD_VAR_SUFFIX

Y
Yu Yang 已提交
486

487
def convert_np_dtype_to_dtype_(np_dtype):
488 489
    """
    Convert the data type in numpy to the data type in Paddle
490

491
    Args:
492
        np_dtype(np.dtype): the data type in numpy.
493

494 495
    Returns:
        core.VarDesc.VarType: the data type in Paddle.
496 497

    """
498 499
    dtype = np.dtype(np_dtype)
    if dtype == np.float32:
500
        return core.VarDesc.VarType.FP32
501
    elif dtype == np.float64:
502
        return core.VarDesc.VarType.FP64
503
    elif dtype == np.float16:
504
        return core.VarDesc.VarType.FP16
505
    elif dtype == np.int32:
506
        return core.VarDesc.VarType.INT32
507
    elif dtype == np.int16:
508
        return core.VarDesc.VarType.INT16
509
    elif dtype == np.int64:
510
        return core.VarDesc.VarType.INT64
511
    elif dtype == np.bool:
512
        return core.VarDesc.VarType.BOOL
513 514
    elif dtype == np.uint16:
        return core.VarDesc.VarType.INT16
515 516
    elif dtype == np.uint8:
        return core.VarDesc.VarType.UINT8
Q
qingqing01 已提交
517 518
    elif dtype == np.int8:
        return core.VarDesc.VarType.INT8
519
    else:
M
minqiyang 已提交
520
        raise ValueError("Not supported numpy dtype %s" % dtype)
521 522 523


def dtype_is_floating(dtype):
524 525 526
    """
    Check the data type is floating or not.
    Args:
527
        dtype(np.dtype|core.VarDesc.VarType): data type.
528 529 530 531 532
            Could be numpy format or Paddle format

    Returns(bool): True if data type is a float value

    """
533
    if not isinstance(dtype, core.VarDesc.VarType):
534 535
        dtype = convert_np_dtype_to_dtype_(dtype)

536 537 538 539
    return dtype in [
        core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32,
        core.VarDesc.VarType.FP64
    ]
540 541


Y
Yang Yang(Tony) 已提交
542
def _debug_string_(proto, throw_on_error=True):
543 544 545 546 547 548 549 550 551 552 553
    """
    Get the debug string of a protobuf message. The message could be not
    initialized.
    Args:
        proto(google.protobuf.message.Message): The protobuf message
        throw_on_error(bool): True if raise an error when the protobuf message
            is not initialized.

    Returns(str): The debug string of the protobuf message

    """
Y
Yu Yang 已提交
554
    error_fields = list()
Y
Yang Yang(Tony) 已提交
555
    if not proto.IsInitialized(error_fields) and throw_on_error:
C
caoying03 已提交
556 557
        raise ValueError("{0} are not initialized.\nThe message is {1}:\n".
                         format(error_fields, proto))
Y
Yu Yang 已提交
558 559 560
    return proto.__str__()


X
Xin Pan 已提交
561
class Variable(object):
562
    """
J
Jiabin Yang 已提交
563
    **Notes**:
564
        **The constructor of Variable should not be invoked directly.**
J
Jiabin Yang 已提交
565

566 567
        **In Static Graph Mode: Please use** `Block.create_var` **to create a Static variable which has no data until being feed.**

J
Jiabin Yang 已提交
568 569 570
        **In Dygraph Mode: Please use** :ref:`api_fluid_dygraph_to_variable` **to create a dygraph variable with real data**

    In Fluid, every input and output of an OP is a variable. In most
571
    cases, variables are used for holding different kinds of data or training
J
Jiabin Yang 已提交
572 573
    labels. A variable belongs to a :ref:`api_guide_Block_en` . All variable has its own name and
    two variables in different :ref:`api_guide_Block_en` could have the same name.
574

575
    There are many kinds of variables. Each kind of them has its own attributes
J
Jiabin Yang 已提交
576
    and usages. Please refer to the `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_ for details.
577

578
    Most of a Variable's member variables can be setted to be None. It mean
579
    it is not available or will be specified later.
580

581
    Examples:
582 583
        In Static Graph Mode:

584 585
        .. code-block:: python

586
            import paddle.fluid as fluid
587
            cur_program = fluid.Program()
588 589 590 591
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
J
Jiabin Yang 已提交
592
        In `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_  Mode:
593 594 595 596 597 598 599 600 601

        .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                new_variable = fluid.dygraph.to_variable(np.arange(10))

602 603
    """

Y
Yu Yang 已提交
604 605
    def __init__(self,
                 block,
Y
Yu Yang 已提交
606
                 type=core.VarDesc.VarType.LOD_TENSOR,
Y
Yu Yang 已提交
607 608 609 610
                 name=None,
                 shape=None,
                 dtype=None,
                 lod_level=None,
611
                 capacity=None,
Q
QI JUN 已提交
612
                 persistable=None,
F
fengjiayi 已提交
613
                 error_clip=None,
Y
Yu Yang 已提交
614
                 stop_gradient=False,
F
fengjiayi 已提交
615
                 is_data=False,
H
Huihuang Zheng 已提交
616
                 need_check_feed=False,
H
hong 已提交
617
                 belong_to_optimizer=False,
Y
Yu Yang 已提交
618
                 **kwargs):
Y
Yu Yang 已提交
619 620
        self.block = block
        if name is None:
Y
Yu Yang 已提交
621
            name = unique_name.generate('_generated_var')
D
Dong Zhihong 已提交
622

Y
Yu Yang 已提交
623
        if dtype is not None:
624
            if not isinstance(dtype, core.VarDesc.VarType):
625
                dtype = convert_np_dtype_to_dtype_(dtype)
626

H
hong 已提交
627 628
        self.belong_to_optimizer = belong_to_optimizer

L
lujun 已提交
629
        if in_dygraph_mode():
M
minqiyang 已提交
630
            # record vars in tracer rather than blocks
M
minqiyang 已提交
631
            self._ivar = kwargs.get("ivar", None)
632
            self.stop_gradient_ = kwargs.get("stop_gradient", True)
M
minqiyang 已提交
633
            if not self._ivar:
634
                self._ivar = core.VarBase(
J
Jiabin Yang 已提交
635 636 637
                    name, type
                    if type else core.VarDesc.VarType.LOD_TENSOR, dtype
                    if dtype else core.VarDesc.VarType.FP32,
638
                    list(shape) if shape else [], True
X
fix  
Xin Pan 已提交
639
                    if persistable else False)
M
minqiyang 已提交
640
            if persistable:
L
lujun 已提交
641
                _dygraph_tracer().trace_var(name, self)
M
minqiyang 已提交
642
            self.op = None
M
minqiyang 已提交
643
        else:
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707
            self.error_clip = error_clip

            is_new_var = False
            name = cpt.to_text(name)
            self.desc = self.block.desc.find_var(cpt.to_bytes(name))

            if self.desc is None:
                self.desc = self.block.desc.var(cpt.to_bytes(name))
                is_new_var = True

            if is_new_var:
                self.desc.set_type(type)
            elif self.desc.type() != type:
                raise ValueError(
                    "Variable {0} has been created before. The "
                    "previous type is {1}; the new type is {2}. They"
                    " are not matched".format(self.name, self.desc.type(),
                                              type))

            if shape is not None:
                if is_new_var:
                    self.desc.set_shape(shape)
                else:
                    old_shape = self.shape
                    shape = tuple(shape)
                    if shape != old_shape:
                        raise ValueError(
                            "Variable {0} has been created before. the previous "
                            "shape is {1}; the new shape is {2}. They are not "
                            "matched.".format(self.name, old_shape, shape))
            if dtype is not None:
                if is_new_var:
                    self.desc.set_dtype(dtype)
                else:
                    old_dtype = self.dtype
                    if dtype != old_dtype:
                        raise ValueError(
                            "Variable {0} has been created before. "
                            "The previous data type is {1}; the new "
                            "data type is {2}. They are not "
                            "matched.".format(self.name, old_dtype, dtype))

            if lod_level is not None:
                if is_new_var:
                    self.desc.set_lod_level(lod_level)
                else:
                    if lod_level != self.lod_level:
                        raise ValueError(
                            "Variable {0} has been created before. "
                            "The previous lod_level is {1}; the new "
                            "lod_level is {2}. They are not "
                            "matched".format(self.name, self.lod_level,
                                             lod_level))
            if persistable is not None:
                if is_new_var:
                    self.desc.set_persistable(persistable)
                else:
                    if persistable != self.persistable:
                        raise ValueError(
                            "Variable {0} has been created before."
                            "The previous persistable is {1}; the new "
                            "persistable is {2}. They are not matched".format(
                                self.name, self.persistable, persistable))

H
Huihuang Zheng 已提交
708 709 710
            if need_check_feed and is_new_var:
                self.desc.set_need_check_feed(need_check_feed)

711 712 713 714 715 716 717 718
            if capacity is not None:
                if is_new_var:
                    self.desc.set_capacity(capacity)
                else:
                    # TODO(abhinavarora) : Compare with set capacity once,
                    # get_capacity is implemented
                    pass

M
minqiyang 已提交
719
            self.block.vars[name] = self
720
            self.op = None
721
            self._stop_gradient = stop_gradient
722
            self.is_data = is_data
Y
Yu Yang 已提交
723

724
    @dygraph_only
725 726
    def detach(self):
        """
J
Jiabin Yang 已提交
727 728
        **Notes**:
            **This API is ONLY avaliable in Dygraph mode**
729

730
        Returns a new Variable, detached from the current graph.
731

732
        Returns:
J
Jiabin Yang 已提交
733
             ( :ref:`api_guide_Variable_en` | dtype is same as current Variable): The detached Variable.
734

735

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
                from paddle.fluid.dygraph import FC
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
                    fc = FC("fc", 64, num_flatten_dims=2)
                    data = to_variable(data)
                    x = fc(data)
                    y = x.detach()

        """
        if in_dygraph_mode():
            new_var = self._cloneVar()
            self.block.append_op(
                type="assign",
                inputs={'X': [self]},
                outputs={'Out': [new_var]},
                stop_gradient=True)
            return new_var
        else:
            raise AttributeError("static graph model DO NOT supprt detach")

763
    @dygraph_only
764
    def numpy(self):
765
        """
J
Jiabin Yang 已提交
766 767
        **Notes**:
            **This API is ONLY avaliable in Dygraph mode**
768

J
Jiabin Yang 已提交
769
        Returns a numpy array shows the value of current :ref:`api_guide_Variable_en`
770 771 772 773 774

        Returns:
            ndarray: The numpy value of current Variable.

        Returns type:
J
Jiabin Yang 已提交
775
            ndarray: dtype is same as current Variable
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
                from paddle.fluid.dygraph import FC
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
                    fc = FC("fc", 64, num_flatten_dims=2)
                    data = to_variable(data)
                    x = fc(data)
                    print(x.numpy())

        """

        if not self._ivar.value().get_tensor()._is_initialized():
            raise ValueError("%s is Empty, Please check if it has no data in" %
                             self.name)
M
minqiyang 已提交
797
        new_ivar = self._ivar._copy_to(core.CPUPlace(), True)
P
Paddle CI 已提交
798
        return np.array(new_ivar.value().get_tensor())
799

800 801 802
    @dygraph_only
    def set_value(self, value):
        """
J
Jiabin Yang 已提交
803 804 805
        **Notes**:
            **This API is ONLY avaliable in Dygraph mode**

806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
        Set a new value for this Variable.

        Args:
            value (Variable|np.ndarray): the new value.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
                from paddle.fluid.dygraph import FC
                import numpy as np

                data = np.ones([3, 32, 32], dtype='float32')
                with fluid.dygraph.guard():
                    fc = fluid.dygraph.FC("fc", 4)
                    t = to_variable(data)
                    fc(t)  # call with default weight
                    custom_weight = np.random.randn(1024, 4).astype("float32")
                    fc.weight.set_value(custom_weight)  # change existing weight
                    out = fc(t)  # call with different weight

        """
H
hong 已提交
829 830 831 832
        assert isinstance(value, (Variable, np.ndarray, core.VarBase)), \
                "Variable set_value function, arguments type only support Variable, numpy, VarBase"

        value_np = value
833
        if isinstance(value, Variable):
H
hong 已提交
834 835 836 837 838 839 840 841 842 843 844 845 846 847
            value_np = value.numpy()
        elif isinstance(value, core.VarBase):
            value_np = _var_base_to_np(value)
        self_tensor = self._ivar.value().get_tensor()

        self_tensor_np = np.array(self_tensor)

        assert self_tensor_np.shape == value_np.shape,  \
                                      "Variable Shape not match, Variable [ {} ] need tensor with shape {} but load set tensor with shape {}".format( self._ivar.name, self_tensor_np.shape, value_np.shape)

        assert self_tensor_np.dtype == value_np.dtype,  \
                                      "Variable dtype not match, Variable [ {} ] need tensor with dtype {}  but load tensor with dtype {}".format( self._ivar.name, self_tensor_np.dtype, value_np.dtype)

        self_tensor.set(value_np, _current_expected_place())
848

849
    @dygraph_only
850
    def backward(self, backward_strategy=None):
851
        """
J
Jiabin Yang 已提交
852 853
        **Notes**:
            **This API is ONLY avaliable in Dygraph mode**
854 855 856

        Run backward of current Graph which starts from current Variable

J
Jiabin Yang 已提交
857 858
        Args:
            backward_strategy( :ref:`api_fluid_dygraph_BackwardStrategy` ): The Backward Strategy to run backward
859

J
Jiabin Yang 已提交
860 861
        Returns:
            NoneType: None
862 863 864 865 866 867 868 869 870 871 872 873

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
J
Jiabin Yang 已提交
874 875
                        # if we don't set tmp's stop_gradient as False then, all path to loss will has no gradient since
                        # there is no one need gradient on it.
876 877 878 879 880 881 882 883 884
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)

        """
J
Jiabin Yang 已提交
885 886 887 888 889
        if in_dygraph_mode():
            from .dygraph import BackwardStrategy
            if backward_strategy is None:
                backward_strategy = BackwardStrategy()
                backward_strategy.sort_sum_gradient = False
890

J
Jiabin Yang 已提交
891 892 893 894
            self._ivar._run_backward(backward_strategy, _dygraph_tracer())
        else:
            raise ValueError(
                "Variable.backward() is only avaliable in DyGraph mode")
895

896
    @dygraph_only
897
    def gradient(self):
898
        """
J
Jiabin Yang 已提交
899 900
        **Notes**:
            **This API is ONLY avaliable in Dygraph mode**
901 902 903

        Get the Gradient of Current Variable

J
Jiabin Yang 已提交
904 905
        Returns:
            ndarray: Numpy value of the gradient of current Variable
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
                    print(loss2.gradient())

        """
        if self._ivar._grad_ivar() is None:
            raise ValueError("%s has no grad, Please set Variable.stop_gradient=False, or " \
                             "check if this is the first and only variable need grad, if so, please set its pre-Variable's " \
                             "stop_gradient=False, to make sure it has gradient " % self.name)
        if not self._ivar._grad_ivar().value().get_tensor()._is_initialized():
            raise ValueError(
                "%s's Grad is Empty, Please check if it has no data in" %
                self.name)
936 937
        new_ivar = self._ivar._grad_ivar()._copy_to(core.CPUPlace(), True)
        return np.array(new_ivar.value().get_tensor())
938

939
    @dygraph_only
940
    def clear_gradient(self):
941
        """
J
Jiabin Yang 已提交
942 943 944 945
        **Notes**:
            **1. This API is ONLY avaliable in Dygraph mode**

            **2. Use it only Variable has gradient, normally we use this for Parameters since other temporal Variable will be deleted by Python's GC**
946

J
Jiabin Yang 已提交
947
        Clear  (set to ``0`` ) the Gradient of Current Variable
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973

        Returns:  None

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
                    print(loss2.gradient())
                    loss2.clear_gradient()
                    print("After clear {}".format(loss2.gradient()))

        """
X
Xin Pan 已提交
974
        self._ivar._clear_gradient()
X
Xin Pan 已提交
975

976
    def __str__(self):
Y
Yang Yang(Tony) 已提交
977 978
        return self.to_string(True)

F
update  
fengjiayi 已提交
979
    def to_string(self, throw_on_error, with_details=False):
980 981 982
        """
        Get debug string.

J
Jiabin Yang 已提交
983 984 985 986 987
        Args:

            throw_on_error (bool): True if raise an exception when self is not initialized.

            with_details (bool): more details about variables and parameters (e.g. trainable, optimize_attr, ...) will be printed when with_details is True. Default value is False;
988

989 990
        Returns:
            str: The debug string.
991 992 993 994 995

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
996

997 998 999 1000 1001
                cur_program = fluid.Program()
                cur_block = cur_program.current_block()
                new_variable = cur_block.create_var(name="X",
                                                    shape=[-1, 23, 48],
                                                    dtype='float32')
1002
                print(new_variable.to_string(True))
J
Jiabin Yang 已提交
1003
                print("=============with detail===============")
1004
                print(new_variable.to_string(True, True))
1005
        """
L
lujun 已提交
1006
        if in_dygraph_mode():
L
lujun 已提交
1007
            # TODO(panyx0718): add more dygraph debug info.
J
Jiabin Yang 已提交
1008 1009 1010 1011 1012 1013 1014
            tensor = self._ivar.value().get_tensor()
            if tensor._is_initialized():
                return 'name %s, dtype: %s shape: %s %s' % (
                    self.name, self.dtype, self.shape, str(tensor))
            else:
                return 'name %s, shape: %s, not inited' % (self.name,
                                                           self.shape)
1015

F
update  
fengjiayi 已提交
1016 1017
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
1018
        protostr = self.desc.serialize_to_string()
1019
        proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr))
F
update  
fengjiayi 已提交
1020 1021 1022 1023
        res_str = _debug_string_(proto, throw_on_error)
        if with_details:
            additional_attr = ("error_clip", "stop_gradient")
            for attr_name in additional_attr:
1024 1025 1026
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))

F
update  
fengjiayi 已提交
1027
        return res_str
1028 1029 1030

    __repr__ = __str__

1031
    @property
1032
    def stop_gradient(self):
J
Jiabin Yang 已提交
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
        """
        Indicating if we stop gradient from current Variable

        **Notes: This Property has default value as** ``True`` **in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, while Parameter's default value is False. However, in Static Graph Mode all Variable's default stop_gradient value is** ``False``

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                value0 = np.arange(26).reshape(2, 13).astype("float32")
                value1 = np.arange(6).reshape(2, 3).astype("float32")
                value2 = np.arange(10).reshape(2, 5).astype("float32")
                fc = fluid.FC("fc1", size=5, dtype="float32")
                fc2 = fluid.FC("fc2", size=3, dtype="float32")
                a = fluid.dygraph.to_variable(value0)
                b = fluid.dygraph.to_variable(value1)
                c = fluid.dygraph.to_variable(value2)
                out1 = fc(a)
                out2 = fc2(b)
                out1.stop_gradient = True
                out = fluid.layers.concat(input=[out1, out2, c], axis=1)
                out.backward()

                assert (fc._w.gradient() == 0).all()
                assert (out1.gradient() == 0).all()
        """
L
lujun 已提交
1062
        if in_dygraph_mode():
M
minqiyang 已提交
1063 1064
            return self._ivar.stop_gradient
        else:
1065
            return self._stop_gradient
1066

1067 1068
    @stop_gradient.setter
    def stop_gradient(self, s):
L
lujun 已提交
1069
        if in_dygraph_mode():
M
minqiyang 已提交
1070
            self._ivar.stop_gradient = s
1071
        else:
1072
            self._stop_gradient = s
1073

1074 1075
    @property
    def persistable(self):
J
Jiabin Yang 已提交
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
        """
        Indicating if we current Variable should be long-term alive


        **Notes: This Property will be deprecated and this API is just to help user understand concept**

            **1. All Variable's persistable is** ``False`` **except Parameters.**

            **2. In** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, this property should not be changed**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("persistable of current Var is: {}".format(new_variable.persistable))
        """
L
lujun 已提交
1097
        if in_dygraph_mode():
1098 1099 1100
            return self._ivar.persistable
        else:
            return self.desc.persistable()
1101

Y
Yu Yang 已提交
1102 1103
    @persistable.setter
    def persistable(self, p):
L
lujun 已提交
1104
        if in_dygraph_mode():
1105 1106 1107
            logging.warn(
                "There will be no use to set persistable in Dygraph Mode, since "
                "you can just do it by hold it as normal Python variable")
1108 1109
        else:
            self.desc.set_persistable(p)
Y
Yu Yang 已提交
1110

Y
Yu Yang 已提交
1111 1112
    @property
    def name(self):
J
Jiabin Yang 已提交
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
        """
        Indicating name of current Variable

        **Notes: If it has two or more Varaible share the same name in the same** :ref:`api_guide_Block_en` **, it means these Variable will share content in no-** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode. This is how we achieve Parameter sharing**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("name of current Var is: {}".format(new_variable.name))
        """
L
lujun 已提交
1129
        if in_dygraph_mode():
1130 1131 1132
            return self._ivar.name
        else:
            return cpt.to_text(self.desc.name())
Y
Yu Yang 已提交
1133

T
typhoonzero 已提交
1134 1135
    @name.setter
    def name(self, new_name):
L
lujun 已提交
1136
        if in_dygraph_mode():
1137 1138 1139
            self._ivar.name = new_name
        else:
            self.desc.set_name(new_name)
T
typhoonzero 已提交
1140

Y
Yu Yang 已提交
1141 1142
    @property
    def shape(self):
J
Jiabin Yang 已提交
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
        """
        Indicating shape of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("shape of current Var is: {}".format(new_variable.shape))

        """
Y
Yu Yang 已提交
1160
        # convert to tuple, make it as same as numpy API.
L
lujun 已提交
1161
        if in_dygraph_mode():
1162 1163 1164
            return self._ivar.shape
        else:
            return tuple(self.desc.shape())
Y
Yu Yang 已提交
1165 1166

    @property
F
fengjiayi 已提交
1167
    def dtype(self):
J
Jiabin Yang 已提交
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
        """
        Indicating data type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Dtype of current Var is: {}".format(new_variable.dtype))
        """
L
lujun 已提交
1184
        if in_dygraph_mode():
1185 1186 1187
            return self._ivar.dtype
        else:
            return self.desc.dtype()
Y
Yu Yang 已提交
1188 1189

    @property
1190
    @dygraph_not_support
Y
Yu Yang 已提交
1191
    def lod_level(self):
J
Jiabin Yang 已提交
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
        """
        Indicating ``LoD`` info of current Variable, please refer to  :ref:`api_fluid_LoDTensor_en` to check the meaning
        of ``LoD``

        **Notes**:

            **1. This is a read-only property**

            **2. Don't support this property in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, it's value should be** ``0(int)``

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("LoD Level of current Var is: {}".format(new_variable.lod_level))
        """
L
lujun 已提交
1213
        # TODO(minqiyang): Support lod_level in dygraph mode
H
Hongyu Liu 已提交
1214 1215
        if in_dygraph_mode():
            raise Exception("Dygraph model DO NOT supprt lod")
1216
        return self.desc.lod_level()
Y
Yu Yang 已提交
1217

Y
Yu Yang 已提交
1218 1219
    @property
    def type(self):
J
Jiabin Yang 已提交
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
        """
        Indicating Type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Type of current Var is: {}".format(new_variable.type))
        """
L
lujun 已提交
1236
        if in_dygraph_mode():
J
Jiabin Yang 已提交
1237
            return self._ivar.type
1238 1239
        else:
            return self.desc.type()
Y
Yu Yang 已提交
1240

W
Wu Yi 已提交
1241
    def _set_error_clip(self, error_clip):
1242 1243 1244 1245 1246 1247 1248 1249 1250
        """
        Set the error_clip.

        Args:
            error_clip(BaseErrorClipAttr) : The new error_clip.

        Returns:
            None
        """
1251 1252
        self.error_clip = error_clip

1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
    def _slice_indices(self, slice, length):
        """
        Reference implementation for the slice.indices method.
        """
        # Compute step and length as integers.
        step = 1 if slice.step is None else slice.step

        # Raise ValueError for negative length or zero step.
        if length < 0:
            raise ValueError("length should not be negative")
        if step == 0:
            raise ValueError("slice step cannot be zero")

        # Find lower and upper bounds for start and stop.
        lower = -1 if step < 0 else 0
        upper = length - 1 if step < 0 else length

        # Compute start.
        if slice.start is None:
            start = upper if step < 0 else lower
        else:
            start = slice.start
            start = max(start + length, lower) if start < 0 else min(start,
                                                                     upper)

        # Compute stop.
        if slice.stop is None:
            stop = lower if step < 0 else upper
        else:
            stop = slice.stop
            stop = max(stop + length, lower) if stop < 0 else min(stop, upper)

        return start, stop, step

    def _detectEllipsis(self, item):
        has_ellipsis = False
        start = 0
        end = len(self.shape)
        for index, o in enumerate(item):
            if o is Ellipsis:
                if has_ellipsis:
                    raise ValueError("Index can have one ellipsis only.")
                has_ellipsis = True
                start = index
            else:
                if has_ellipsis:
                    end = index
        return has_ellipsis, start, end

    def _reconstructSliceinfo(self, item):
        has_ellipsis, start, end = self._detectEllipsis(item)
        if has_ellipsis:
            newitem = []
            for i in range(start):
                newitem.append(item[i])
            for i in range(start, end):
                newitem.append(slice(None, None, None))
            for i in range(end, len(item)):
                newitem.append(item[i])
            return newitem
        else:
            return None

    def _detectContinuesSlice(self, item):
        starts = []
        ends = []
        for index, o in enumerate(item):
            if isinstance(o, int):
                start = int(o)
                if (index > 0 and index >= self.shape[index]) \
                        or (index < 0 and (index + self.shape[index]) < 0):
                    raise IndexError("invalid index")
                start = max(start + self.shape[index], 0) if start < 0 else min(
                    start, self.shape[index])
                starts.append(start)
                ends.append(start + 1)
            elif isinstance(o, slice):
                start, stop, step = self._slice_indices(o, self.shape[index])
                if step == 1 or step == -1:
                    starts.append(start)
                    ends.append(stop)
                else:
                    return False, None
            else:
                raise IndexError("Valid index accept int or slice or ellipsis")
        return True, [starts, ends]

L
lujun 已提交
1340
    def _cloneVar(self, copy=False):
1341 1342
        if not copy:
            return self.block.create_var(
H
Hongyu Liu 已提交
1343 1344
                name=unique_name.generate_with_ignorable_key(self.name),
                dtype=self.dtype)
1345 1346 1347 1348
        else:
            return self

    def _sliceVar(self, axes, starts, ends):
L
lujun 已提交
1349
        new_var = self._cloneVar()
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
        self.block.append_op(
            type="slice",
            inputs={'Input': [self]},
            outputs={'Out': [new_var]},
            attrs={'axes': axes,
                   'starts': starts,
                   'ends': ends})
        return new_var

    def _concatVar(self, inputs, axis):
L
lujun 已提交
1360
        new_var = self._cloneVar()
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
        self.block.append_op(
            type="concat",
            inputs={'X': inputs},
            outputs={'Out': [new_var]},
            attrs={'axis': axis, })
        return new_var

    def _sliceAndConcatVar(self, item, axis):
        if isinstance(item, slice):
            if self.shape[axis] < 0:
L
lujun 已提交
1371
                return self._cloneVar(True)
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
            start, stop, step = self._slice_indices(item, self.shape[axis])
            if step == 1:
                return self._sliceVar([axis], [start], [stop])
            else:
                vars = []
                if step > 0:
                    while start < stop:
                        vars.append(
                            self._sliceVar([axis], [start], [start + 1]))
                        start += step
                else:
                    while start > stop:
                        vars.append(
                            self._sliceVar([axis], [start], [start + 1]))
                        start += step
                return self._concatVar(vars, axis)
        elif isinstance(item, int):
            if self.shape[axis] < 0:
L
lujun 已提交
1390
                return self._cloneVar(True)
1391
            index = int(item)
1392
            if (index > 0 and index >= self.shape[axis]) \
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
                    or (index < 0 and (index + self.shape[axis]) < 0):
                raise IndexError("invalid index")
            return self._sliceVar([axis], [index], [index + 1])
        else:
            raise IndexError("Valid index accept int or slice or tuple")

    def __getitem__(self, item):
        """
        Slice the variable.

        Args:
            item(int/slice/tuple) : the index.

        Returns:
            Sliced variable
        """
H
Hongyu Liu 已提交
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418

        if not isinstance(item, tuple):
            item = [item]

        decrease_axis = []
        slice_axis = []
        slice_start = []
        slice_end = []
        reverse_axis = []

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
        def fill_constant(shape, dtype, value, force_cpu=False, out=None):
            self.block.append_op(
                type='fill_constant',
                inputs={},
                outputs={'Out': [out]},
                attrs={
                    'shape': shape,
                    'dtype': out.dtype,
                    'value': float(value),
                    'force_cpu': force_cpu or force_init_on_cpu()
                },
                stop_gradient=True)
            out.stop_gradient = True
            return out

H
Hongyu Liu 已提交
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
        for dim, slice_item in enumerate(item):
            if isinstance(slice_item, slice):
                start = slice_item.start
                end = slice_item.stop
                step = slice_item.step if slice_item.step else 1

                assert (step == 1 or step == -1)

                if step == -1:
                    reverse_axis.append(dim)
                    assert (start is None and end is None)

                if start is None and end is None:
                    continue

                if start is None:
                    start = 0

                if end is None:
                    end = 10000000

                slice_axis.append(dim)
                slice_start.append(start)
                slice_end.append(end)
1458
            else:
H
Hongyu Liu 已提交
1459 1460 1461
                decrease_axis.append(dim)
                slice_axis.append(dim)
                slice_start.append(slice_item)
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
                if isinstance(slice_item, Variable):
                    temp_1 = self.block.create_var(dtype='int32')
                    fill_constant([1], 'int32', 1, force_cpu=True, out=temp_1)
                    temp_end = self.block.create_var(dtype='int32')
                    self.block.append_op(
                        type='elementwise_add',
                        inputs={'X': slice_item,
                                'Y': temp_1},
                        outputs={'Out': temp_end},
                        attrs={'axis': -1})
                    slice_end.append(temp_end)
                else:
                    slice_end.append(slice_item + 1
                                     if slice_item != -1 else 10000000)

        def contain_var(one_list):
            for ele in one_list:
                if isinstance(ele, Variable):
                    return True
            return False

        def get_new_list_tensor(old_list):
            new_list_tensor = []
            for dim in old_list:
                if isinstance(dim, Variable):
                    dim.stop_gradient = True
                    new_list_tensor.append(dim)
                else:
                    assert (isinstance(dim, int))
                    temp_out = self.block.create_var(dtype='int32')
                    fill_constant(
                        [1], 'int32', dim, force_cpu=True, out=temp_out)
                    new_list_tensor.append(temp_out)
            return new_list_tensor

        inputs = {'Input': [self]}
        attrs = {
            'axes': slice_axis,
            'starts': [],
            'ends': [],
            'decrease_axis': decrease_axis
        }
        infer_flags = list(1 for i in range(len(slice_axis)))

        # starts
        if not contain_var(slice_start):
            attrs['starts'] = slice_start
        else:
            inputs['StartsTensorList'] = get_new_list_tensor(slice_start)
            for i, dim in enumerate(slice_start):
                if isinstance(dim, Variable):
                    attrs['starts'].append(-1)
                    infer_flags[i] = -1
                else:
                    attrs['starts'].append(dim)
        # ends
        if not contain_var(slice_end):
            attrs['ends'] = slice_end
        else:
            inputs['EndsTensorList'] = get_new_list_tensor(slice_end)
            for i, dim in enumerate(slice_end):
                if isinstance(dim, Variable):
                    attrs['ends'].append(-1)
                    infer_flags[i] = -1
                else:
                    attrs['ends'].append(dim)
        # infer_flags
        attrs['infer_flags'] = infer_flags
H
Hongyu Liu 已提交
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540

        out = self
        if len(slice_axis) > 0:
            # append slice_op here
            slice_out_var = self.block.create_var(
                name=unique_name.generate_with_ignorable_key(self.name +
                                                             "_slice"),
                dtype=self.dtype)

            self.block.append_op(
                type="slice",
1541
                inputs=inputs,
H
Hongyu Liu 已提交
1542
                outputs={'Out': [slice_out_var]},
1543
                attrs=attrs)
H
Hongyu Liu 已提交
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560

            out = slice_out_var

        if len(reverse_axis) > 0:
            reverse_out_var = self.block.create_var(
                name=unique_name.generate_with_ignorable_key(self.name +
                                                             "_slice_reverse"),
                dtype=self.dtype)
            self.block.append_op(
                type="reverse",
                inputs={'X': out},
                outputs={'Out': [reverse_out_var]},
                attrs={'axis': reverse_axis})

            out = reverse_out_var

        return out
1561

Y
Yu Yang 已提交
1562

F
fengjiayi 已提交
1563 1564 1565
def get_all_op_protos():
    """
    Get all registered op proto from PaddlePaddle C++ end.
1566

1567 1568
    Returns:
       list: list of OpProto.
F
fengjiayi 已提交
1569 1570 1571 1572
    """
    protostrs = core.get_all_op_protos()
    ret_values = []
    for pbstr in protostrs:
1573
        op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr))
F
fengjiayi 已提交
1574 1575 1576 1577 1578
        ret_values.append(op_proto)
    return ret_values


class OpProtoHolder(object):
1579 1580 1581 1582
    """
    A global variable to hold all OpProtos from C++ as a map
    """

F
fengjiayi 已提交
1583 1584 1585 1586 1587 1588 1589 1590 1591
    @classmethod
    def instance(cls):
        if not hasattr(cls, '_instance'):
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        assert not hasattr(
            self.__class__,
1592
            '_instance'), 'Please use `instance()` to get OpProtoHolder object!'
F
fengjiayi 已提交
1593 1594 1595 1596 1597 1598
        op_protos = get_all_op_protos()
        self.op_proto_map = {}
        for proto in op_protos:
            self.op_proto_map[proto.type] = proto

    def get_op_proto(self, type):
1599 1600 1601 1602 1603 1604 1605 1606
        """
        Get OpProto by a type string.
        Args:
            type(str): The type that operator registered in C++ side.

        Returns(framework_pb2.OpProto): The OpProto

        """
Y
Yu Yang 已提交
1607 1608
        if type not in self.op_proto_map:
            raise ValueError("Operator \"%s\" has not been registered." % type)
F
fengjiayi 已提交
1609 1610
        return self.op_proto_map[type]

1611 1612 1613 1614 1615 1616
    def update_op_proto(self):
        op_protos = get_all_op_protos()
        for proto in op_protos:
            if proto.type not in self.op_proto_map:
                self.op_proto_map[proto.type] = proto

1617 1618 1619 1620
    @staticmethod
    def generated_op_attr_names():
        return {
            core.op_proto_and_checker_maker.kOpRoleAttrName(),
S
sneaxiy 已提交
1621
            core.op_proto_and_checker_maker.kOpRoleVarAttrName(),
1622 1623
            core.op_proto_and_checker_maker.kOpNameScopeAttrName(),
            core.op_proto_and_checker_maker.kOpCreationCallstackAttrName()
1624 1625
        }

F
fengjiayi 已提交
1626

X
Xin Pan 已提交
1627
class Operator(object):
1628
    """
1629 1630 1631 1632 1633 1634 1635
    In Fluid, all the operation are represented by Operator, and Operator
    is regarded as a build in an instruction of a Block. Users can use the
    build in instructions to describe their neural network.

    Args:
        block(Block): The block has the current operator.
        desc(core.OpDesc): The protobuf description of Operator.
C
chengduoZH 已提交
1636
        type(str): The type of operator. Default None.
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
        inputs(dict): The input of this Operator. it is a dictionary, for every
            element, key is the input parameter name, and value is a list of
            variables. Default None.
        outputs(dict): The output of this Operator. it is a dictionary, for
            every element, key is the input parameter name, and value is a list
            of variables. Default None.
        attrs(dict): The attributes of this Operator. it is a dictionary, for
            every element, key is attribute name, and value is the attribute value.
            The attribute type should be as same as the type registered in C++ side.
            Default None.

    Returns:
        Operator: The initialized Operator.

    Raises:
        ValueError: If the passed input, output and attrs doesn't match the
            initializing Operator's that registered in C++ side.

    Notes:
        The constructor of operator should not be invoked directly. Use
W
Wu Yi 已提交
1657
        Block.append_op or Block._prepend_op instead.
1658 1659 1660 1661

    Examples:
        .. code-block:: python

1662
            import paddle.fluid as fluid
1663
            cur_program = fluid.Program()
1664 1665 1666 1667 1668
            cur_block = cur_program.current_block()
            # var1 += var2 + var3
            cur_block.append_op(type="sum",
                                inputs={"X": [var1, var2, var3]},
                                outputs={"Out": [var1]})
1669
    """
1670
    OP_WITHOUT_KERNEL_SET = {
1671 1672
        'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad',
        'conditional_block', 'while', 'send', 'recv', 'listen_and_serv',
1673 1674
        'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify',
        'gen_nccl_id', 'c_gen_nccl_id', 'c_comm_init', 'c_sync_calc_stream',
1675
        'c_sync_comm_stream'
1676
    }
1677

Y
Yu Yang 已提交
1678 1679
    def __init__(self,
                 block,
Y
Yu Yang 已提交
1680
                 desc,
Y
Yu Yang 已提交
1681 1682 1683
                 type=None,
                 inputs=None,
                 outputs=None,
M
minqiyang 已提交
1684
                 attrs=None):
L
lujun 已提交
1685
        if in_dygraph_mode():
1686 1687
            if type is None:
                raise ValueError(
1688
                    "`type` to initialized an Operator can not be None.")
J
Jiabin Yang 已提交
1689
            self._type = type
M
minqiyang 已提交
1690
            self.attrs = attrs if attrs else {}
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704
        else:
            self.block = block
            self.desc = desc
            # note: not add self.attrs here:
            # https://github.com/PaddlePaddle/Paddle/pull/12583#pullrequestreview-145093173
            op_attrs = attrs
            if op_attrs is None:
                op_attrs = dict()
            del attrs

            op_maker = core.op_proto_and_checker_maker

            if op_maker.kOpRoleAttrName() not in op_attrs:
                op_attrs[op_maker.kOpRoleAttrName(
1705
                )] = self.block.program._op_role
1706 1707 1708

            role_var_name = op_maker.kOpRoleVarAttrName()
            if len(self.block.program.
1709 1710
                   _op_role_var) != 0 and role_var_name not in op_attrs:
                op_attrs[role_var_name] = self.block.program._op_role_var
1711 1712 1713 1714 1715 1716 1717 1718

            if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
                del op_attrs[role_var_name]

            if len(self.desc.type()) != 0:
                return
            if type is None:
                raise ValueError(
1719
                    "`type` to initialized an Operator can not be None.")
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
            else:
                callstack_var_name = op_maker.kOpCreationCallstackAttrName()
                op_attrs[callstack_var_name] = list(
                    reversed(traceback.format_stack()))[1:]

            self.desc.set_type(type)
            proto = OpProtoHolder.instance().get_op_proto(type)

            namescope_var_name = op_maker.kOpNameScopeAttrName()
            op_attrs[namescope_var_name] = _full_name_scope()

            def find_name(var_list, name):
                for var_name in var_list:
                    if var_list[var_name] is not None and var_name == name:
                        return True
                return False

            if inputs is not None:
                for in_proto in proto.inputs:
                    found = find_name(inputs, in_proto.name)
                    assert found or in_proto.dispensable, "Input {} not found".format(
                        in_proto.name)
                    if found:
                        in_args = inputs[in_proto.name]
                        if not isinstance(in_args, list):
                            in_args = [in_args]
                        if not in_proto.duplicable and len(in_args) > 1:
                            raise ValueError(
                                "Input %s expects only one input, but %d are given."
                                % (in_proto.name, len(in_args)))
                        in_arg_names = []
1751
                        for index, arg in enumerate(in_args):
1752 1753 1754 1755
                            if isinstance(arg, six.string_types):
                                in_arg_names.append(arg)
                            elif isinstance(arg, six.binary_type):
                                in_arg_names.append(arg.decode())
1756
                            elif isinstance(arg, Variable):
1757
                                in_arg_names.append(cpt.to_text(arg.name))
1758 1759 1760 1761
                            else:
                                raise ValueError(
                                    "not suprt args type , should be[ string_type, binary_type, Varibale]"
                                )
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
                        self.desc.set_input(in_proto.name, in_arg_names)
                    else:
                        self.desc.set_input(in_proto.name, [])

            if outputs is not None:
                for m in proto.outputs:
                    if (m.name not in outputs) and m.dispensable:
                        continue
                    if not ((m.name in outputs) or m.dispensable):
                        raise ValueError(("Incorrect setting for output(s) of "
                                          "operator \"%s\", should set: [%s].")
                                         % (type, m.name))
                for out_proto in proto.outputs:
                    if out_proto.name not in outputs:
                        continue
                    out_args = outputs[out_proto.name]
                    if not isinstance(out_args, list):
                        out_args = [out_args]
                    if not out_proto.duplicable and len(out_args) > 1:
                        raise ValueError(
                            "Output %s expects only one output, but %d are given."
                            % (out_proto.name, len(out_args)))
                    out_arg_names = []
                    for arg in out_args:
                        out_arg_names.append(cpt.to_text(arg.name))
                        # TODO(minqiyang): could we remove variable's op in static mode?
L
lujun 已提交
1788
                        if not in_dygraph_mode():
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807
                            arg.op = self
                    self.desc.set_output(out_proto.name, out_arg_names)

            if op_attrs is not None:
                if not isinstance(op_attrs, dict):
                    raise TypeError("'attrs' should be a dict.")
                for attr in proto.attrs:
                    attr_name = attr.name
                    if (attr_name not in op_attrs) or (
                            op_attrs[attr_name] is None):
                        continue
                    attr_val = op_attrs[attr_name]
                    self._update_desc_attr(attr_name, attr_val)

            self.desc.check_attrs()
            if self._has_kernel(type):
                self.desc.infer_var_type(self.block.desc)
                self.desc.infer_shape(self.block.desc)

W
Wu Yi 已提交
1808
    def _has_kernel(self, op_type):
1809 1810
        return op_type not in self.OP_WITHOUT_KERNEL_SET

Y
Yang Yang(Tony) 已提交
1811
    def to_string(self, throw_on_error):
1812
        """
1813 1814
        Get debug string.

1815
        Args:
1816 1817
            throw_on_error(bool): Whether to raise exception if self is not
                initialized.
1818

1819 1820
        Returns:
            str: The debug string.
1821 1822

        """
1823
        protostr = self.desc.serialize_to_string()
1824
        proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr))
Y
Yang Yang(Tony) 已提交
1825 1826 1827 1828
        return _debug_string_(proto, throw_on_error)

    def __str__(self):
        return self.to_string(True)
1829 1830 1831

    __repr__ = __str__

F
fengjiayi 已提交
1832 1833
    @property
    def type(self):
L
lujun 已提交
1834
        if in_dygraph_mode():
J
Jiabin Yang 已提交
1835
            return self._type
1836 1837
        else:
            return self.desc.type()
F
fengjiayi 已提交
1838 1839

    def input(self, name):
1840
        """
1841
        Get the input arguments according to the input parameter name.
1842

1843 1844
        Args:
            name(str): The input parameter name.
1845

1846 1847 1848
        Returns:
            list: return the list of argument names that associated with \
                the specific parameter name.
1849
        """
F
fengjiayi 已提交
1850 1851
        return self.desc.input(name)

W
Wu Yi 已提交
1852
    def _rename_input(self, old_name, new_name):
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's input.
            new_name(str): The new name of the Operator's input.

        Returns:
            None
        """
W
Wu Yi 已提交
1863
        self.desc._rename_input(old_name, new_name)
T
typhoonzero 已提交
1864

W
Wu Yi 已提交
1865
    def _rename_output(self, old_name, new_name):
1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's output.
            new_name(str): The new name of the Operator's output.

        Returns:
            None
        """
W
Wu Yi 已提交
1876
        self.desc._rename_output(old_name, new_name)
T
typhoonzero 已提交
1877

F
fengjiayi 已提交
1878 1879 1880 1881
    @property
    def input_names(self):
        return self.desc.input_names()

T
typhoonzero 已提交
1882 1883 1884 1885 1886 1887 1888 1889
    @property
    def input_arg_names(self):
        return self.desc.input_arg_names()

    @property
    def output_arg_names(self):
        return self.desc.output_arg_names()

F
fengjiayi 已提交
1890
    def output(self, name):
1891
        """
1892
        Get output arguments by the output parameter name.
1893

1894 1895
        Args:
            name(str): The output parameter name.
1896

1897 1898 1899
        Returns:
            list: return the list of argument names associated with \
                the specific parameter name.
1900
        """
F
fengjiayi 已提交
1901 1902 1903 1904 1905 1906
        return self.desc.output(name)

    @property
    def output_names(self):
        return self.desc.output_names()

1907 1908 1909 1910 1911 1912 1913 1914
    @property
    def idx(self):
        for i, op in enumerate(self.block.ops):
            if op == self:
                return i
        raise ValueError(
            "Can't find op itself in it's block. It could be a bug of Paddle.")

F
fengjiayi 已提交
1915
    def has_attr(self, name):
1916
        """
1917 1918
        Whether this Operator has the attribute with name or not.

1919
        Args:
1920
            name(str): the attribute name.
1921

1922 1923
        Returns:
            bool: True if has this attribute.
1924 1925

        """
F
fengjiayi 已提交
1926 1927 1928
        return self.desc.has_attr(name)

    def attr_type(self, name):
1929
        """
1930
        Get the type of attribute by attribute's name.
1931

1932 1933
        Args:
            name(str): the attribute name.
1934

1935 1936
        Returns:
            core.AttrType: the attribute type.
1937
        """
F
fengjiayi 已提交
1938 1939
        return self.desc.attr_type(name)

W
Wu Yi 已提交
1940
    def _set_attr(self, name, val):
1941 1942 1943 1944 1945 1946 1947 1948 1949 1950
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
G
gongweibao 已提交
1951 1952
        self._update_desc_attr(name, val)

1953 1954 1955
    def _remove_attr(self, name):
        self.desc.remove_attr(name)

G
gongweibao 已提交
1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
    def _update_desc_attr(self, name, val):
        """
        Update the value of desc's attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
Q
Qiyang Min 已提交
1967 1968
        if isinstance(val, Block):
            self.desc.set_block_attr(name, val.desc)
Y
Yancey1989 已提交
1969 1970
        elif isinstance(val, list) and val and all(
                isinstance(v, Block) for v in val):
1971
            self.desc.set_blocks_attr(name, [v.desc for v in val])
Q
Qiyang Min 已提交
1972 1973 1974 1975
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            self.desc.set_serialized_attr(name, val.serialize_to_string())
        else:
W
Wu Yi 已提交
1976
            self.desc._set_attr(name, val)
Y
yuyang18 已提交
1977

F
fengjiayi 已提交
1978 1979 1980 1981 1982
    @property
    def attr_names(self):
        return self.desc.attr_names()

    def attr(self, name):
1983
        """
1984 1985
        Get the attribute by name.

1986
        Args:
1987
            name(str): the attribute name.
1988

1989 1990
        Returns:
            bool|int|str|float|list: The attribute value. The return value
1991 1992
            can be any valid attribute type.
        """
F
fengjiayi 已提交
1993
        return self.desc.attr(name)
Y
Yu Yang 已提交
1994

W
Wu Yi 已提交
1995
    def _block_attr_id(self, name):
1996
        """
G
gongweibao 已提交
1997
        Get the block attribute's id by name.
1998

1999 2000
        Args:
            name(str): the attribute name.
2001

2002 2003
        Returns:
            int: the block index.
2004
        """
W
Wu Yi 已提交
2005
        return self.desc._block_attr_id(name)
G
gongweibao 已提交
2006

W
Wu Yi 已提交
2007
    def _block_attr(self, name):
G
gongweibao 已提交
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017
        """
        Get the block attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            block: the block attribute.
        """

W
Wu Yi 已提交
2018
        id = self._block_attr_id(name)
G
gongweibao 已提交
2019 2020 2021
        assert (id >= 0 and id < len(self.block.program.blocks))
        return self.block.program.blocks[id]

W
Wu Yi 已提交
2022
    def _blocks_attr(self, name):
G
gongweibao 已提交
2023 2024 2025 2026 2027 2028 2029 2030 2031 2032
        """
        Get the blocks attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks attribute.
        """
        attrs = []
W
Wu Yi 已提交
2033
        for i in self._blocks_attr_ids(name):
G
gongweibao 已提交
2034 2035 2036 2037 2038
            assert (i >= 0 and i < len(self.block.program.blocks))
            attrs.append(self.block.program.blocks[i])

        return attrs

W
Wu Yi 已提交
2039
    def _blocks_attr_ids(self, name):
G
gongweibao 已提交
2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
        """
        Get the blocks attribute's ids by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks ids.
        """

W
Wu Yi 已提交
2050
        return self.desc._blocks_attr_ids(name)
Y
Yu Yang 已提交
2051

J
JiayiFeng 已提交
2052
    def all_attrs(self):
F
fengjiayi 已提交
2053
        """
2054 2055 2056
        Get the attribute dict.

        Returns:
G
gongweibao 已提交
2057
            dict: The Operator's attribute dict, name->attr.
F
fengjiayi 已提交
2058 2059 2060 2061
        """
        attr_names = self.attr_names
        attr_map = {}
        for n in attr_names:
G
gongweibao 已提交
2062 2063
            attr_type = self.desc.attr_type(n)
            if attr_type == core.AttrType.BLOCK:
W
Wu Yi 已提交
2064
                attr_map[n] = self._block_attr(n)
G
gongweibao 已提交
2065 2066 2067
                continue

            if attr_type == core.AttrType.BLOCKS:
W
Wu Yi 已提交
2068
                attr_map[n] = self._blocks_attr(n)
G
gongweibao 已提交
2069 2070 2071 2072
                continue

            attr_map[n] = self.attr(n)

F
fengjiayi 已提交
2073 2074
        return attr_map

Y
Yu Yang 已提交
2075

Y
Yu Yang 已提交
2076
class Block(object):
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
    """
    In Fluid, a Program is consistence of multi-Block, and Block stores
    VarDesc and OpDesc. In a specific Block, a VarDesc have a unique name.
    One block could have some child blocks, and child block's name scopes
    should inherit the parent's so that OpDesc in child block can reference
    a VarDesc that is stored in the parent block.
    Please reference the framework.proto for details.

    Args:
        program(Program): The Program that the Block belongs to.
        idx(int): The block's id in the Program.

    Notes:
        The constructor of Block should not be invoked directly. Please
W
Wu Yi 已提交
2091
        use `Program._create_block()` to create a block.
2092 2093 2094 2095

    Examples:
        .. code-block:: python

2096 2097 2098
            import paddle.fluid as fluid

            cur_program = fluid.Program()
2099 2100 2101 2102 2103 2104 2105 2106 2107
            cur_block = cur_program.current_block()
            var = cur_block.create_var(name="X",
                                       shape=[-1, 23, 48],
                                       dtype='float32')
            cur_block.append_op(type="abs",
                                inputs={"X": [var]},
                                outputs={"Out": [var]})
    """

Y
Yu Yang 已提交
2108
    def __init__(self, program, idx):
Y
Yu Yang 已提交
2109
        self.desc = program.desc.block(idx)
2110
        self.vars = collections.OrderedDict()  # var_name --> var
Q
qiaolongfei 已提交
2111
        self.ops = list()  # operator list
Y
Yu Yang 已提交
2112
        self.program = program
2113
        self.removed_vars = collections.OrderedDict()
Y
Yu Yang 已提交
2114

2115
    def __str__(self):
Y
Yang Yang(Tony) 已提交
2116 2117
        return self.to_string(True)

F
fengjiayi 已提交
2118 2119
    def to_string(self, throw_on_error, with_details=False):
        """
2120 2121
        Get debug string.

F
fengjiayi 已提交
2122 2123
        Args:
            throw_on_error(bool): raise exception when self is not initialized
2124
                when throw_on_error is True.
F
update  
fengjiayi 已提交
2125
            with_details(bool): more details about variables and parameters
2126 2127
                (e.g. trainable, optimize_attr, ...) will be printed when
                with_details is True. Default False.
F
fengjiayi 已提交
2128

2129 2130
        Returns:
            str: The debug string.
F
fengjiayi 已提交
2131 2132 2133 2134
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
F
fengjiayi 已提交
2135
            re_add_indent = re.compile(r"\n(.)")
F
fengjiayi 已提交
2136 2137
            res_str = "blocks {\n  idx: %d\n  parent_idx: %d" % (
                self.idx, self.parent_idx)
2138
            for var in list(self.vars.values()):
F
fengjiayi 已提交
2139
                res_str += "\n  vars {\n    %s  }" % re_add_indent.sub(
F
update  
fengjiayi 已提交
2140
                    r"\n    \1", var.to_string(throw_on_error, with_details))
F
fengjiayi 已提交
2141
            for op in self.ops:
F
fengjiayi 已提交
2142 2143
                res_str += "\n  ops {\n    %s  }" % re_add_indent.sub(
                    r"\n    \1", op.to_string(throw_on_error))
F
fengjiayi 已提交
2144 2145 2146
            res_str += "\n}"
        else:
            protostr = self.desc.serialize_to_string()
2147 2148
            proto = framework_pb2.BlockDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
2149 2150
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
2151 2152 2153

    __repr__ = __str__

Y
Yu Yang 已提交
2154 2155
    @property
    def parent_idx(self):
Y
Yu Yang 已提交
2156
        return self.desc.parent
Y
Yu Yang 已提交
2157

Y
Yu Yang 已提交
2158 2159 2160 2161
    @property
    def forward_block_idx(self):
        return self.desc.get_forward_block_idx()

W
Wu Yi 已提交
2162
    def _set_forward_block_idx(self, idx):
2163 2164 2165 2166 2167 2168 2169 2170 2171
        """
        Set the forward block Idx.

        Args:
            idx(int): the block index.

        Returns:
            None
        """
W
Wu Yi 已提交
2172
        self.desc._set_forward_block_idx(idx)
Y
Yu Yang 已提交
2173

Y
Yu Yang 已提交
2174 2175
    @property
    def idx(self):
Y
Yu Yang 已提交
2176
        return self.desc.id
Y
Yu Yang 已提交
2177

Q
Qiao Longfei 已提交
2178
    def var(self, name):
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
        """
        Get a Variable by name from this block.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: The If input's type is not str, or this block
                doesn't have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
2192
        if not isinstance(name, six.string_types):
M
minqiyang 已提交
2193 2194 2195
            raise TypeError(
                "var require string as parameter, but get %s instead." %
                (type(name)))
Y
Yu Yang 已提交
2196 2197
        v = self.vars.get(name, None)
        if v is None:
Q
Qiao Longfei 已提交
2198
            raise ValueError("var %s not in this block" % name)
Y
Yu Yang 已提交
2199
        return v
Q
Qiao Longfei 已提交
2200

X
Xin Pan 已提交
2201
    def _find_var_recursive(self, name):
2202 2203 2204 2205 2206 2207 2208
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Returns:
X
Xin Pan 已提交
2209
            Variable: the Variable with the giving name. Or None if not found.
2210
        """
Y
Yu Yang 已提交
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234
        frontier = list()
        visited = set()

        frontier.append(self)

        prog = self.program

        while len(frontier) != 0:  # BFS
            cur = frontier[0]
            frontier = frontier[1:]

            if id(cur) in visited:
                continue

            if cur.has_var(name):
                return cur.var(name)

            if cur.parent_idx != -1:
                frontier.append(prog.block(cur.parent_idx))

            if cur.forward_block_idx != -1:
                frontier.append(prog.block(cur.forward_block_idx))

            visited.add(id(cur))
X
Xin Pan 已提交
2235
        return None
Y
Yu Yang 已提交
2236

X
Xin Pan 已提交
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255
    def _var_recursive(self, name):
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: this block and this parent block doesn't
                have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
        var = self._find_var_recursive(name)
        if var:
            return var
        else:
            raise ValueError("Var {0} is not found recursively".format(name))
F
fengjiayi 已提交
2256

Q
Qiao Longfei 已提交
2257
    def all_parameters(self):
2258
        return list(self.iter_parameters())
2259

2260
    def iter_parameters(self):
M
minqiyang 已提交
2261
        return (item[1] for item in six.iteritems(self.vars)
2262
                if isinstance(item[1], Parameter))
Q
Qiao Longfei 已提交
2263

Y
Yu Yang 已提交
2264
    def create_var(self, *args, **kwargs):
2265
        var = Variable(block=self, *args, **kwargs)
2266 2267
        if 'initializer' in kwargs:
            kwargs['initializer'](var, self)
Q
Qiao Longfei 已提交
2268
        return var
Y
Yu Yang 已提交
2269

Q
Qiao Longfei 已提交
2270 2271 2272
    def has_var(self, name):
        return name in self.vars

W
Wu Yi 已提交
2273
    def _rename_var(self, name, new_name):
T
typhoonzero 已提交
2274 2275
        """
        Rename variable in vars and ops' inputs and outputs
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287

        Args:
            name(str): the name that need to be renamed.
            new_name(str): the name that need to rename to.

        Raises:
            ValueError: If this block doesn't have this the giving name,
                or the type of the var with the giving name is not Parameter
                or Variable.

        Returns:
            Variable: the Variable with the giving name.
T
typhoonzero 已提交
2288
        """
M
minqiyang 已提交
2289 2290
        name = cpt.to_text(name)
        new_name = cpt.to_text(new_name)
M
minqiyang 已提交
2291

T
typhoonzero 已提交
2292
        if not self.has_var(name):
2293
            raise ValueError("var %s is not in current block" % name)
T
wip  
typhoonzero 已提交
2294 2295
        v = self.var(name)
        if type(v) == Parameter:
T
typhoonzero 已提交
2296
            var_type = "Parameter"
T
wip  
typhoonzero 已提交
2297 2298 2299 2300 2301 2302 2303
            stop_gradient = v.stop_gradient
            trainable = v.trainable
            optimize_attr = v.optimize_attr
            regularizer = v.regularizer
            gradient_clip_attr = v.gradient_clip_attr
            error_clip = v.error_clip
        elif type(v) == Variable:
T
typhoonzero 已提交
2304
            var_type = "Variable"
T
wip  
typhoonzero 已提交
2305 2306 2307 2308
            error_clip = v.error_clip
            stop_gradient = v.stop_gradient
        else:
            raise ValueError("unsupported var type: %s", type(v))
T
typhoonzero 已提交
2309
        orig_var_type = v.type
M
minqiyang 已提交
2310
        self.desc._rename_var(cpt.to_bytes(name), cpt.to_bytes(new_name))
W
Wu Yi 已提交
2311
        # NOTE: v is destroyed by C++ after calling _rename_var.
M
minqiyang 已提交
2312
        d = self.desc.find_var(cpt.to_bytes(new_name))
T
typhoonzero 已提交
2313
        if var_type == "Parameter":
T
wip  
typhoonzero 已提交
2314 2315 2316 2317
            var = Parameter(
                self,
                d.shape(),
                d.dtype(),
T
typhoonzero 已提交
2318
                type=orig_var_type,
T
wip  
typhoonzero 已提交
2319 2320 2321 2322 2323 2324 2325
                name=new_name,
                stop_gradient=stop_gradient,
                trainable=trainable,
                optimize_attr=optimize_attr,
                regularizer=regularizer,
                gradient_clip_attr=gradient_clip_attr,
                error_clip=error_clip)
T
typhoonzero 已提交
2326
        elif var_type == "Variable":
T
wip  
typhoonzero 已提交
2327 2328
            var = Variable(
                self,
T
typhoonzero 已提交
2329
                type=orig_var_type,
T
wip  
typhoonzero 已提交
2330 2331 2332 2333
                name=new_name,
                error_clip=error_clip,
                stop_gradient=stop_gradient)

W
Wu Yi 已提交
2334
        # rename the python side, _sync_with_cpp will only add
T
wip  
typhoonzero 已提交
2335 2336 2337
        # new vars/ops to python side.
        self.vars[new_name] = var
        del self.vars[name]
W
Wu Yi 已提交
2338
        self._sync_with_cpp()
2339
        return var
T
typhoonzero 已提交
2340

W
Wu Yi 已提交
2341 2342
    def _remove_var(self, name):
        self._sync_with_cpp()
M
minqiyang 已提交
2343
        self.desc._remove_var(cpt.to_bytes(name))
2344 2345
        del self.vars[name]

Y
Yu Yang 已提交
2346 2347
    def create_parameter(self, *args, **kwargs):
        global_block = self.program.global_block()
Q
Qiao Longfei 已提交
2348
        param = Parameter(global_block, *args, **kwargs)
2349
        if 'initializer' in kwargs:
2350 2351 2352 2353 2354

            def _is_inited_by(block, var):
                init_ops = []
                for op in block.ops:
                    if var.name in op.output_arg_names:
2355 2356 2357 2358 2359
                        # In startup_program, "c_broadcast" and "c_sync_comm_stream"
                        # are treated as initialization ops that cause error. 
                        # Think of "c_broadcast" and "c_sync_comm_stream" as a special case here.
                        if op.type in ["c_broadcast", "c_sync_comm_stream"]:
                            continue
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374
                        init_ops.append(op)
                return init_ops

            initializer = kwargs['initializer']
            init_ops = _is_inited_by(global_block, param)
            init_ops_len = len(init_ops)
            if init_ops_len > 1:
                raise RuntimeError("param " + param.name +
                                   " is inited by multiple init ops " + str(
                                       init_ops))
            elif init_ops_len == 1:
                #TODO already inited, do nothing, should log a warning
                pass
            else:
                initializer(param, self)
2375
        param.stop_gradient = False
Q
Qiao Longfei 已提交
2376
        return param
Y
Yu Yang 已提交
2377

Y
Yu Yang 已提交
2378
    def append_op(self, *args, **kwargs):
2379 2380 2381 2382 2383 2384
        """
        Appends a new Operator according to the giving arguments.

        Returns:
            Operator: the append Operator.
        """
L
lujun 已提交
2385
        if in_dygraph_mode():
2386 2387 2388
            attrs = kwargs.get("attrs", {})
            if _dygraph_tracer_._train_mode == False:
                # eval mode
2389 2390 2391 2392 2393
                if ('trainable_statistics' not in attrs
                    ) or not attrs['trainable_statistics']:
                    attrs['is_test'] = True
                else:
                    attrs['is_test'] = False
2394

J
Jiabin Yang 已提交
2395 2396
            type = kwargs.get("type", None)

2397 2398 2399
            op = Operator(
                block=self,
                desc=None,
J
Jiabin Yang 已提交
2400
                type=type,
M
minqiyang 已提交
2401 2402
                inputs=None,
                outputs=None,
2403
                attrs=attrs)
2404

M
minqiyang 已提交
2405 2406 2407
            # record ops in tracer rather than blocks
            #
            # TODO(minqiyang): add op stop_gradient support in static mode too.
L
lujun 已提交
2408
            # currently, we only support stop_gradient in dygraph mode.
J
Jiabin Yang 已提交
2409 2410

            _dygraph_tracer().trace_op(type,
M
minqiyang 已提交
2411
                                       kwargs.get("inputs", {}),
J
Jiabin Yang 已提交
2412 2413
                                       kwargs.get("outputs", {}), attrs
                                       if attrs else {},
M
minqiyang 已提交
2414
                                       kwargs.get("stop_gradient", False))
M
minqiyang 已提交
2415
        else:
2416 2417 2418 2419 2420 2421 2422 2423 2424
            op_desc = self.desc.append_op()
            op = Operator(
                block=self,
                desc=op_desc,
                type=kwargs.get("type", None),
                inputs=kwargs.get("inputs", None),
                outputs=kwargs.get("outputs", None),
                attrs=kwargs.get("attrs", None))

M
minqiyang 已提交
2425
            self.ops.append(op)
M
minqiyang 已提交
2426

2427 2428
        return op

W
Wu Yi 已提交
2429
    def _insert_op(self, index, *args, **kwargs):
2430 2431 2432 2433 2434 2435 2436 2437 2438
        """
        Insert a Operator according to the giving arguments.

        Args:
            index(int): the place that the operator to insert.

        Returns:
            Operator: the insert Operator.
        """
W
Wu Yi 已提交
2439 2440
        self._sync_with_cpp()
        op_desc = self.desc._insert_op(index)
Q
qiaolongfei 已提交
2441 2442 2443 2444
        op = Operator(block=self, desc=op_desc, *args, **kwargs)
        self.ops.insert(index, op)
        return op

W
Wu Yi 已提交
2445
    def _remove_op(self, index):
2446 2447 2448 2449 2450 2451 2452 2453 2454
        """
        Remove the specific position operator.

        Args:
            index(int): the position that the operator to insert.

        Returns:
            None
        """
W
Wu Yi 已提交
2455 2456
        self._sync_with_cpp()
        self.desc._remove_op(index, index + 1)
2457 2458
        del self.ops[index]

W
Wu Yi 已提交
2459
    def _slice_ops(self, start, end):
2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
        """
        Return the Operator between start and end.

        Args:
            start(int): the start position.
            end(int): the end position.

        Returns:
            list: the Operators between start and end.
        """
Q
qiaolongfei 已提交
2470
        return self.ops[start:end]
Y
Yancey1989 已提交
2471

W
Wu Yi 已提交
2472
    def _prepend_op(self, *args, **kwargs):
L
lujun 已提交
2473
        if in_dygraph_mode():
J
Jiabin Yang 已提交
2474 2475
            type = kwargs.get("type", None)
            attrs = kwargs.get("attrs", {})
2476
            op = Operator(
J
Jiabin Yang 已提交
2477
                self, None, type=type, inputs=None, outputs=None, attrs=attrs)
M
minqiyang 已提交
2478

J
Jiabin Yang 已提交
2479
            _dygraph_tracer().trace_op(type,
M
minqiyang 已提交
2480
                                       kwargs.get("inputs", {}),
J
Jiabin Yang 已提交
2481 2482
                                       kwargs.get("outputs", {}), attrs
                                       if attrs else {},
M
minqiyang 已提交
2483
                                       kwargs.get("stop_gradient", False))
M
minqiyang 已提交
2484
        else:
2485 2486 2487 2488 2489 2490 2491 2492
            op_desc = self.desc._prepend_op()
            op = Operator(
                self,
                op_desc,
                type=kwargs.get("type", None),
                inputs=kwargs.get("inputs", None),
                outputs=kwargs.get("outputs", None),
                attrs=kwargs.get("attrs", None))
M
minqiyang 已提交
2493
            self.ops.insert(0, op)
2494

Y
Yu Yang 已提交
2495 2496
        return op

W
Wu Yi 已提交
2497
    def _sync_with_cpp(self):
2498
        """
2499 2500
        Sync from the desc on the c++ end. This method is used to synchronize
        the c++ desc instance generated by backward.
2501
        """
Q
Qiao Longfei 已提交
2502 2503 2504 2505 2506
        # sync variables from cpp
        for var in self.desc.all_vars():
            if not self.has_var(var.name()):
                self.create_var(name=var.name(), desc=var, type=var.type())

2507
        # sync variables removed from c++ end
2508
        for var in list(self.vars.keys()):
M
minqiyang 已提交
2509
            if not self.desc.find_var(cpt.to_bytes(var)):
2510 2511
                self.vars.pop(var)

Q
Qiao Longfei 已提交
2512
        # sync operators from cpp
2513 2514 2515 2516
        ops_in_cpp = []
        for op_idx in range(0, self.desc.op_size()):
            ops_in_cpp.append(self.desc.op(op_idx))

Y
Yu Yang 已提交
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
        if len(self.ops) != 0:
            first_op_in_python = self.ops[0].desc
            last_op_in_python = self.ops[len(self.ops) - 1].desc
            start_index = None
            end_index = None
            for index in range(len(ops_in_cpp)):
                if first_op_in_python == ops_in_cpp[index]:
                    start_index = index
                if last_op_in_python == ops_in_cpp[index]:
                    end_index = index
            assert start_index is not None
            assert end_index is not None
            assert start_index <= end_index
        else:
            start_index = 0
            end_index = -1
Q
Qiao Longfei 已提交
2533 2534 2535 2536 2537

        # sync ops append to the head of cpp_ops
        for index in range((start_index - 1 - 1), -1, -1):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
Q
qiaolongfei 已提交
2538
            self.ops.insert(0, op)
Q
Qiao Longfei 已提交
2539 2540 2541 2542 2543 2544 2545

        # sync ops append to the end of cpp_ops
        for index in range((end_index + 1), len(ops_in_cpp)):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.append(op)

2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
        # sync ops removed from c++ end
        if end_index != -1 and end_index < len(self.ops):
            ops_in_cpp_index = 0
            ops_in_python_index = 0
            while ops_in_python_index < len(
                    self.ops) and ops_in_cpp_index < len(ops_in_cpp):
                if self.ops[ops_in_python_index].desc != ops_in_cpp[
                        ops_in_cpp_index]:
                    del self.ops[ops_in_python_index]
                else:
                    ops_in_cpp_index += 1
                    ops_in_python_index += 1

Q
Qiao Longfei 已提交
2559 2560 2561 2562
        assert len(self.ops) == len(ops_in_cpp)
        for index in range(len(self.ops)):
            assert self.ops[index].desc == ops_in_cpp[index]

W
Wu Yi 已提交
2563
    def _copy_param_info_from(self, other):
2564
        """
2565 2566
        Copy the information of parameters from the other block.

2567
        Args:
2568 2569 2570 2571 2572
            other(Block): the other block.

        Raises:
            ValueError: If type of input is not Block, or the `other` and this
                block is not in the same topology.
2573 2574 2575 2576 2577

        Returns:
            None
        """
        if not isinstance(other, Block):
W
Wu Yi 已提交
2578 2579
            raise TypeError(
                "_copy_param_info_from should be invoked with Block")
2580
        for p in other.iter_parameters():
2581 2582 2583
            assert isinstance(p, Parameter)
            v = self.vars.get(p.name, None)
            if v is None:
W
Wu Yi 已提交
2584
                raise ValueError("_copy_param_info_from should be invoked with "
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
                                 "same topology")
            assert isinstance(v, Variable)
            new_p = Parameter(
                block=self,
                shape=v.shape,
                dtype=v.dtype,
                type=v.type,
                lod_level=v.lod_level,
                stop_gradient=p.stop_gradient,
                trainable=p.trainable,
                optimize_attr=p.optimize_attr,
                regularizer=p.regularizer,
F
fengjiayi 已提交
2597
                gradient_clip_attr=p.gradient_clip_attr,
F
fengjiayi 已提交
2598
                error_clip=p.error_clip,
2599 2600 2601
                name=v.name)
            self.vars[new_p.name] = new_p

2602
    def _clone_variable(self, var, force_persistable=True):
2603 2604
        """
        Clone a variable into current block.
2605

2606 2607
        Args:
            var: the variable to be cloned.
2608 2609 2610
            force_persistable(bool): True means setting the result variable to being persistable.
                                     False means setting the persistable the same with that of input var.
                                     default: True.
2611 2612

        Returns:
2613
            Variable: the new  variable cloned from 'var' in current block.
2614 2615
        """
        assert isinstance(var, Variable)
T
update  
typhoonzero 已提交
2616 2617 2618 2619 2620
        ret_var = None
        # make STEP_SCOPES var can be safely cloned.
        if var.type == core.VarDesc.VarType.STEP_SCOPES:
            ret_var = self.create_var(
                name=var.name, persistable=var.persistable, type=var.type)
T
tangwei12 已提交
2621 2622
        elif var.type == core.VarDesc.VarType.RAW:
            ret_var = self.create_var(
T
tangwei12 已提交
2623
                name=var.name, persistable=var.persistable, type=var.type)
T
typhoonzero 已提交
2624 2625 2626 2627 2628 2629
        elif var.type == core.VarDesc.VarType.SELECTED_ROWS:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
2630
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
2631 2632
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
2633 2634 2635 2636 2637 2638 2639
        else:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
                lod_level=var.lod_level,
2640
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
2641 2642
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
2643
        return ret_var
2644

Y
Yu Yang 已提交
2645

2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740
class IrNode(object):
    """
    Python IrNode. Beneath it is a core.Node, which is used for Ir Pass.
    """

    def __init__(self, node):
        """
        Construct an IrNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node,
                          core.Node), 'node must be the instance of core.Node.'
        self.node = node

    def name(self):
        """
        Return the node name.

        Returns:
            str: node name.
        """
        return self.node.name()

    def node_type(self):
        """
        Return the node type.

        Returns:
            core.Node.Type: node type(core.Node.Type.Operation or core.Node.Type.Variable).
        """
        return self.node.node_type()

    def var(self):
        """
        Return the node variable description.

        Returns:
            core.VarDesc: node variable description.
        """
        return self.node.var()

    def op(self):
        """
        Return the node operator description.

        Returns:
            core.OpDesc: node operator description.
        """
        return self.node.op()

    def id(self):
        """
        Return the node id.

        Returns:
            int: node id.
        """
        return self.node.id()

    def is_op(self):
        """
        If the node is an operator, then return true.

        Returns:
            bool: indicate whether the node is an operator.
        """
        return self.node.is_op()

    def is_var(self):
        """
        If the node is a variable, then return true.

        Returns:
            bool: indicate whether the node is a variable.
        """
        return self.node.is_var()

    def is_ctrl_var(self):
        """
        If the node is a control dependence variable, then return true.

        Returns:
            bool: indicate whether the node is a control dependence variable.
        """
        return self.node.is_ctrl_var()

    def clear_inputs(self):
        """
        Clear the node inputs. After executing the `clear_inputs` function,
        the node inputs will be empty.
        """
        self.node.clear_inputs()

2741
    def remove_input_by_id(self, node_id):
2742 2743 2744 2745 2746 2747
        """
        Remove a node from inputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
2748
        self.node.remove_input(node_id)
2749

2750
    def remove_input(self, node):
2751 2752 2753 2754
        """
        Remove a node from inputs.

        Args:
2755
            node(IrNode): the node being removed.
2756
        """
2757
        self.node.remove_input(node.node)
2758

2759
    def append_input(self, node):
2760 2761 2762 2763
        """
        Append a node in inputs.

        Args:
2764
            node(IrNode): the node being appended.
2765
        """
2766
        self.node.append_input(node.node)
2767 2768 2769 2770 2771 2772 2773 2774

    def clear_outputs(self):
        """
        Clear the node outputs. After executing the `clear_outputs` function,
        the node outputs will be empty.
        """
        self.node.clear_outputs()

2775
    def remove_output_by_id(self, node_id):
2776 2777 2778 2779 2780 2781
        """
        Remove a node from outputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
2782
        self.node.remove_output(node_id)
2783

2784
    def remove_output(self, node):
2785 2786 2787 2788
        """
        Remove a node from outputs.

        Args:
2789
            node(IrNode): the node being removed.
2790
        """
2791
        self.node.remove_output(node.node)
2792

2793
    def append_output(self, node):
2794 2795 2796 2797
        """
        Append a node in outputs.

        Args:
2798
            node(IrNode): the node being appended.
2799
        """
2800
        self.node.append_output(node.node)
2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861

    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrNode): node inputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrNode): node outputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.outputs]


class IrVarNode(IrNode):
    """
    Python IrVarNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrVarNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_var(), \
            'node must be the instance of core.Node and it must be a variable node.'
        super(IrVarNode, self).__init__(node)
        self.node = node

    def set_shape(self, shape):
        """
        Set the node variable shape.

        Args:
            shape(list): shape to be set.
        """
        assert self.node.var() is not None, \
            "The node variable description cannot be None."
        self.node.var().set_shape(shape)

    def persistable(self):
        """
        If the variable node is a persistable variable, then return true.

        Returns:
            bool: indicate whether the variable is persistable.
        """
        assert self.node.var() is not None, \
            "The node variable description cannot be None."
        return self.node.var().persistable()

2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
    def type(self):
        """
        Return the variable type.

        Returns:
            core.VarDesc.VarType: the variable type.
        """
        assert self.node.var() is not None, \
            "The node variable description cannot be None."
        return self.node.var().type()

    def dtype(self):
        """
        Return the variable data type.

        Returns:
            core.VarDesc.VarType: the variable data type.
        """
        assert self.node.var() is not None, \
            "The node variable description cannot be None."
        return self.node.var().dtype()

    def shape(self):
        """
        Return the variable shape.

        Returns:
            list: the variable shape.
        """
        assert self.node.var() is not None, \
            "The node variable description cannot be None."
        return self.node.var().shape()

2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrOpNode): node inputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrOpNode): node outputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.outputs]


class IrOpNode(IrNode):
    """
    Python IrOpNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrOpNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_op(), \
            'node must be the instance of core.Node and it must be a operator node.'
        super(IrOpNode, self).__init__(node)
        self.node = node

    def rename_input(self, old_input_name, new_input_name):
        """
        Rename the input of this node.

        Args:
            old_input_name(str): the old input name.
            new_input_name(str): the new input name.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        self.node.op()._rename_input(old_input_name, new_input_name)

2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958
    def rename_output(self, old_output_name, new_output_name):
        """
        Rename the output of this node.

        Args:
            old_output_name(str): the old output name.
            new_output_name(str): the new output name.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        print("op: {}, old: {}, new: {}\n".format(self.node.op().type(
        ), old_output_name, new_output_name))
        self.node.op()._rename_output(old_output_name, new_output_name)

2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997
    def input(self, name):
        """
        Get the argument name list by the parameter name for input.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        return self.node.op().input(name)

    def output(self, name):
        """
        Get the argument name list by the parameter name for output.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        return self.node.op().output(name)

    def set_type(self, new_type):
        """
        Change the operator type into new type.

        Args:
            new_type(str): new operator type to be set.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        return self.node.op().set_type(new_type)

2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017
    def set_attr(self, name, val):
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.
        """
        self._update_desc_attr(name, val)

    def _update_desc_attr(self, name, val):
        """
        Update the value of the op desc's attribute by attribute's name.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        desc = self.node.op()
        if isinstance(val, Block):
            desc.set_block_attr(name, val.desc)
        elif isinstance(val, list) and val and \
3018
                all(isinstance(v, Block) for v in val):
3019 3020
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
3021
                isinstance(val, core.ProgramDesc):
3022 3023 3024 3025
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)

3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047
    def input_arg_names(self):
        """
        Return input arguments' names of this op node.

        Returns:
            list(str): input arguments' names of this op node.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        return self.node.op().input_arg_names()

    def output_arg_names(self):
        """
        Return output arguments' names of this op node.

        Returns:
            list(str): output arguments' names of this op node.
        """
        assert self.node.op() is not None, \
            "The node operator description cannot be None."
        return self.node.op().output_arg_names()

3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrVarNode): node inputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrVarNode): node outputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.outputs]


3069 3070
class IrGraph(object):
    """
3071
    Python IrGraph. Beneath it is a core.Graph, which is used for
3072
    creating a c++ Ir Pass Graph. An IrGraph is just a graph view of
3073 3074
    a Program. In an IrGraph, both Variables and Operators are graph
    nodes.
3075 3076 3077 3078
    """

    def __init__(self, graph, for_test=False):
        """
3079 3080
        Construct an IrGraph using core.Graph.

3081 3082 3083 3084 3085 3086 3087 3088 3089
        Args:
            graph(core.Graph): C++ Graph.
            for_test(bool): True for the test graph and false for the train graph.
        """
        assert isinstance(
            graph, core.Graph), 'graph must be the instance of core.Graph.'
        self.graph = graph
        self._for_test = for_test

3090 3091 3092 3093
    def clone(self):
        """
        Create a new and duplicated IrGraph.

3094 3095 3096
        Warns:
            The method only clones the graph structure, not its attributes.

3097 3098 3099
        Returns:
            IrGraph: A new and duplicated graph.
        """
3100
        g = self.graph.clone()
3101 3102
        return IrGraph(g, self._for_test)

3103
    def is_test(self):
3104 3105 3106
        """
        If the graph is used for testing, the function returns true. Otherwise, returns false.
        """
3107 3108
        return self._for_test

W
WangZhen 已提交
3109
    def all_nodes(self):
3110 3111 3112
        """
        Return all nodes included in the graph as a set.
        """
3113
        return {IrNode(node) for node in self.graph.nodes()}
3114

3115
    def all_var_nodes(self):
3116 3117 3118
        """
        Return all variable nodes included in the graph as a set.
        """
3119
        return {IrVarNode(node) for node in self.graph.nodes() if node.is_var()}
3120

3121
    def all_persistable_nodes(self):
3122 3123 3124
        """
        Return all persistable variable nodes included in the graph as a set.
        """
W
WangZhen 已提交
3125 3126 3127 3128 3129
        persistable_nodes = set()
        for node in self.graph.nodes():
            if node.is_var() and node.var() is not None and node.var(
            ).persistable():
                persistable_nodes.add(node)
3130
        return {IrVarNode(p) for p in persistable_nodes}
W
WangZhen 已提交
3131

3132
    def all_op_nodes(self):
3133 3134 3135
        """
        Return all operator nodes included in the graph as a set.
        """
3136
        return {IrOpNode(node) for node in self.graph.nodes() if node.is_op()}
3137

3138
    def create_persistable_node(self, name, var_type, shape, var_dtype):
3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149
        """
        Create a persistable variable node in the graph. In IrGraph,
        it can not distinguish between persistable variables and parameters.

        Args:
            name(str): the name of the persistable variable node.
            vart_type(core.VarDesc.VarType): the type of the persistable variable node.
            shape(list): the shape of the persistable variable node.
            var_dtype(core.VarDesc.VarType): the data type of the persistable variable node.

        Returns:
3150
            IrVarNode: the created persistable variable node.
3151
        """
3152 3153 3154 3155 3156
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
        var_desc.set_persistable(True)
3157
        return IrVarNode(self.graph.create_var_node(var_desc))
3158 3159

    def create_var_node(self, name, var_type, shape, var_dtype):
3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170
        """
        Create a variable node in the graph. The created variable node is
        not persistable.

        Args:
            name(str): the name of the variable node.
            vart_type(core.VarDesc.VarType): the type of the variable node.
            shape(list): the shape of the variable node.
            var_dtype(core.VarDesc.VarType): the data type of the variable node.

        Returns:
3171
            IrVarNode: the created variable node.
3172 3173
        """

3174 3175 3176 3177
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
3178
        return IrVarNode(self.graph.create_var_node(var_desc))
3179 3180

    def create_var_node_from_desc(self, var_desc):
3181 3182 3183 3184 3185 3186 3187 3188
        """
        Create a variable node by using an existing VarDesc in the graph.
        Depend on the giving VarDesc, the created variable node may be persistable.

        Args:
            var_desc(core.VarDesc): the giving variable description.

        Returns:
3189
            IrVarNode: the created variable node.
3190
        """
3191
        return IrVarNode(self.graph.create_var_node(var_desc))
3192 3193

    def create_op_node(self, op_type, attrs, inputs, outputs):
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203
        """
        Create a operator node in the graph.

        Args:
            op_type(str): the type of the operator node.
            attrs(dict): the attributes of the operator node.
            inputs(dict): the inputs of the operator node.
            outputs(dict): the outpus of the operator node.

        Returns:
3204
            IrOpNode: the created operator node.
3205
        """
3206 3207
        op_desc = core.OpDesc()
        op_desc.set_type(op_type)
3208
        for attr, value in six.iteritems(attrs):
3209
            self._update_desc_attr(op_desc, attr, value)
3210
        for input_name, var_nodes in six.iteritems(inputs):
3211 3212 3213 3214
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_input(input_name,
                              [var_node.name() for var_node in var_nodes])
3215
        for output_name, var_nodes in six.iteritems(outputs):
3216 3217 3218 3219
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_output(output_name,
                               [var_node.name() for var_node in var_nodes])
3220
        return IrOpNode(self.graph.create_op_node(op_desc))
3221 3222

    def create_op_node_from_desc(self, op_desc):
3223 3224 3225 3226 3227 3228 3229
        """
        Create a operator node by using an existing OpDesc in the graph.

        Args:
            op_desc(core.VarDesc): the giving operator description.

        Returns:
3230
            IrOpNode: the created operator node.
3231
        """
3232
        return IrOpNode(self.graph.create_op_node(op_desc))
3233 3234

    def update_input_link(self, old_input_node, new_input_node, op_node):
3235 3236 3237 3238
        """
        Update the input's link of a operator node.

        Args:
3239 3240 3241
            old_input_node(IrNode): the old input node of the giving op_node.
            new_input_node(IrNode): the new input node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
3242
        """
3243
        assert old_input_node.node in self.graph.nodes() and new_input_node.node in \
3244 3245
               self.graph.nodes() and op_node.node in self.graph.nodes(), \
            'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.'
3246 3247 3248 3249
        old_input_node.remove_output(op_node)
        op_node.remove_input(old_input_node)
        new_input_node.append_output(op_node)
        op_node.append_input(new_input_node)
3250
        op_node.rename_input(old_input_node.name(), new_input_node.name())
3251

3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
    def update_output_link(self, old_output_node, new_output_node, op_node):
        """
        Update the output's link of an operator node.

        Args:
            old_output_node(IrNode): the old output node of the giving op_node.
            new_output_node(IrNode): the new output node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
        """
        assert old_output_node.node in self.graph.nodes() and new_output_node.node in \
        self.graph.nodes() and op_node.node in self.graph.nodes(), \
        'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.'
        old_output_node.remove_input(op_node)
        op_node.remove_output(old_output_node)
        new_output_node.append_input(op_node)
        op_node.append_output(new_output_node)
        op_node.rename_output(old_output_node.name(), new_output_node.name())

3270
    def link_to(self, node_in, node_out):
3271 3272 3273 3274
        """
        Connect two nodes.

        Args:
3275 3276
            node_in(IrNode): the input node.
            node_out(IrNode): the output node.
3277
        """
3278
        assert node_in.node in self.graph.nodes() and node_out.node in self.graph.nodes(), \
W
WangZhen 已提交
3279
            'The two arguments(node_in&node_out) must be in the graph nodes.'
3280 3281
        node_in.append_output(node_out)
        node_out.append_input(node_in)
3282 3283

    def safe_remove_nodes(self, remove_nodes):
3284 3285 3286 3287 3288 3289 3290
        """
        Remove nodes safely since links connected to these removed nodes are
        also removed.

        Args:
            remove_nodes(set): the nodes prepared to be removed.
        """
3291
        if not isinstance(remove_nodes, set):
W
WangZhen 已提交
3292 3293 3294 3295
            if isinstance(remove_nodes, Iterable):
                remove_nodes = set(remove_nodes)
            else:
                remove_nodes = {remove_nodes}
3296 3297
        original_nodes = {n.node for n in remove_nodes}
        core.graph_safe_remove_nodes(self.graph, original_nodes)
3298

Z
Zhen Wang 已提交
3299 3300 3301 3302 3303 3304 3305 3306
    def resolve_hazard(self):
        ordered_nodes = core.topology_sort(self.graph)
        var_nodes = dict()
        for node in ordered_nodes:
            if node.is_op() and node.op() is not None:
                for each_var_name in node.op().input_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
3307
                            self._find_node_by_name(node.inputs, each_var_name)
Z
Zhen Wang 已提交
3308 3309 3310 3311
                        ]
                for each_var_name in node.op().output_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
3312
                            self._find_node_by_name(node.outputs, each_var_name)
Z
Zhen Wang 已提交
3313 3314 3315
                        ]
                    else:
                        var_nodes[each_var_name].append(
3316 3317
                            self._find_node_by_name(node.outputs,
                                                    each_var_name))
Z
Zhen Wang 已提交
3318 3319
        self.graph.resolve_hazard(var_nodes)

W
WangZhen 已提交
3320
    def has_circle(self):
3321 3322 3323 3324 3325 3326
        """
        Check if the graph has a circle.

        Returns:
            bool: True if the graph has a circle else False.
        """
W
WangZhen 已提交
3327 3328 3329
        return core.has_circle(self.graph)

    def graph_num(self):
3330 3331 3332 3333 3334 3335
        """
        Count the number of unconnected graphs in this graph.

        Returns:
            int: the number of unconnected graphs.
        """
W
WangZhen 已提交
3336 3337 3338
        return core.graph_num(self.graph)

    def topology_sort(self):
3339 3340 3341 3342 3343 3344
        """
        Perform the topology sort operation on the graph.

        Notes: the `graph` cannot contain a circle.

        Returns:
Z
Zhen Wang 已提交
3345
            list(IrNode): nodes in topology order.
3346
        """
3347
        ordered_nodes = core.topology_sort(self.graph)
Z
Zhen Wang 已提交
3348
        return [IrNode(n) for n in ordered_nodes]
W
WangZhen 已提交
3349 3350

    def build_adjacency_list(self):
3351 3352 3353 3354
        """
        Build an adjacency list of operations for the `graph`.

        Returns:
3355
            dict{IrNode: set(IrNode)}: the adjacency list.
3356
        """
3357 3358 3359 3360 3361
        adj_list = core.build_adjacency_list(self.graph)
        wrapped_adj_list = dict()
        for k, v in six.iteritems(adj_list):
            wrapped_adj_list[IrNode(k)] = {IrNode(n) for n in v}
        return wrapped_adj_list
W
WangZhen 已提交
3362

3363 3364 3365 3366 3367 3368 3369 3370
    def draw(self, save_path, name, marked_nodes=None, remove_ctr_var=True):
        """
        Draw the graph. If `dot` command is installed, the drawn graph
        will be saved as pdf file type, otherwise dot file type is used.

        Args:
            save_path(str): the save path of drawn graph.
            name(str): the name of drawn graph.
3371
            marked_nodes(set(IrNode)): nodes that are needed to be marked.
3372 3373 3374 3375 3376
            Default value is None.
            remove_ctr_var(bool): If it is set True, all control variable nodes
            in the graph will be removed. Default value is True.
        """

3377 3378 3379
        def _convert_to_pdf(dot_file_path):
            pdf_save_path = os.path.splitext(dot_file_path)[0] + '.pdf'
            exited_code = subprocess.call('dot -Tpdf ' + dot_file_path \
3380
                                          + ' -o ' + pdf_save_path, shell=True)
3381 3382 3383 3384 3385
            if exited_code != 0:
                print('The dot command is needed for creating pdf files.')
                print('The {} is saved as the dot filetype.'.format(
                    dot_file_path))

3386
        remove_ctr_vars = set()
3387
        if remove_ctr_var:
3388
            for node in self.all_var_nodes():
3389 3390 3391
                if node.is_ctrl_var():
                    remove_ctr_vars.add(node)
            self.safe_remove_nodes(remove_ctr_vars)
3392 3393
        print('Total ops num = {}.'.format(len(self.all_op_nodes())))

3394 3395
        if marked_nodes is not None:
            if not isinstance(marked_nodes, set):
3396 3397 3398 3399 3400 3401
                if isinstance(marked_nodes, Iterable):
                    marked_nodes = set(marked_nodes)
                else:
                    marked_nodes = {marked_nodes}
            marked_nodes = {n.node for n in marked_nodes}
            remove_ctr_vars = {n.node for n in remove_ctr_vars}
3402 3403 3404 3405
            marked_nodes = marked_nodes - remove_ctr_vars
            if self.graph.has('__graphviz__marked_node__'):
                self.graph.erase('__graphviz__marked_node__')
            self.graph.set('__graphviz__marked_node__', marked_nodes)
3406 3407
        if not os.path.exists(save_path):
            os.makedirs(save_path)
3408 3409 3410 3411 3412 3413 3414
        viz_dot_path = os.path.join(save_path, name) + '.dot'
        viz_pass = core.get_pass('graph_viz_pass')
        viz_pass.set('graph_viz_path', viz_dot_path)
        viz_pass.apply(self.graph)
        _convert_to_pdf(viz_dot_path)

    def to_program(self):
3415 3416 3417
        """
        Convert the graph into a Program.

Z
Zhen Wang 已提交
3418
        WARN: When the graph includes backward operator nodes, the
3419 3420 3421 3422 3423 3424
        conversion process may be failed. Usually, this function is
        only used to convert a test graph.

        Returns:
            Program: a program converted from the graph.
        """
3425
        convert_pass = core.get_pass('graph_to_program_pass')
3426 3427
        desc = core.ProgramDesc()
        convert_pass.set_not_owned('program', desc)
3428 3429 3430 3431
        convert_pass.apply(self.graph)
        program = Program._construct_from_desc(desc)
        return program

3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442
    def _find_node_by_name(self, nodes, node_name):
        """
        Find a node in the giving nodes set by the name.
        """
        target_node = None
        for n in nodes:
            if n.name() == node_name:
                target_node = n
        assert target_node is not None, "Cannot find the target node in the giving set."
        return target_node

3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458
    def _update_desc_attr(self, desc, name, val):
        """
        Update the value of desc's attribute by attribute's name.
        """
        if isinstance(val, Block):
            desc.set_block_attr(name, val.desc)
        elif isinstance(val, list) and val and all(
                isinstance(v, Block) for v in val):
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)


Y
Yu Yang 已提交
3459
class Program(object):
D
dzhwinter 已提交
3460
    """
3461 3462
    Create Python Program.  It has at least one :ref:`api_guide_Block_en`, when the
    control flow op like conditional_block, while :ref:`api_fluid_layers_While` is included,
J
Jiabin Yang 已提交
3463
    it will contain nested block.
3464

J
Jiabin Yang 已提交
3465 3466 3467
    Please reference the
    `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_
    for details.
D
dzhwinter 已提交
3468

J
Jiabin Yang 已提交
3469
    A set of Program usually contains startup program and main program.
J
Jiabin Yang 已提交
3470
    A startup program is set to contain some initial work, eg. initialize the ``Parameter``, and the main
J
Jiabin Yang 已提交
3471 3472 3473 3474 3475 3476 3477
    program will contain the network structure and vars for train.

    A set of Program can be used for test or train, in train program ,
    Paddle will contain all content to build a train network,  in test
    program Paddle will prune some content which is irrelevant to test, eg.
    backward ops and vars.

J
Jiabin Yang 已提交
3478 3479 3480 3481
    **Notes**:
        **we have** :ref:`api_fluid_default_startup_program` **and** :ref:`api_fluid_default_main_program`
        **by default, a pair of them will shared the parameters. The** :ref:`api_fluid_default_startup_program` **only run once to initialize parameters,**
        :ref:`api_fluid_default_main_program` **run in every mini batch and adjust the weights.**
D
dzhwinter 已提交
3482 3483

    Returns:
J
Jiabin Yang 已提交
3484
        Program: An empty Program.
D
dzhwinter 已提交
3485 3486

    Examples:
3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499
        .. code-block:: python

            import paddle.fluid as fluid

            main_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(main_program=main_program, startup_program=startup_program):
                x = fluid.layers.data(name="x", shape=[-1, 784], dtype='float32')
                y = fluid.layers.data(name="y", shape=[-1, 1], dtype='int32')
                z = fluid.layers.fc(name="fc", input=x, size=10, act="relu")

            print("main program is: {}".format(main_program))
            print("start up program is: {}".format(startup_program))
D
dzhwinter 已提交
3500 3501 3502

    """

3503 3504
    def __init__(self):
        self.desc = core.ProgramDesc()
Y
Yu Yang 已提交
3505 3506
        self.blocks = [Block(self, 0)]
        self.current_block_idx = 0
D
dzhwinter 已提交
3507
        self._seed = 0
Y
yuyang18 已提交
3508
        self._current_role = core.op_proto_and_checker_maker.OpRole.Forward
3509
        self.__op_role_var = []
T
tangwei12 已提交
3510

3511 3512
        # for distribute training
        # _is_distributed = True if under distributed training
T
tangwei12 已提交
3513
        self._is_distributed = False
3514
        # _is_chief = True if the trainer is the first one, usually No.0
T
tangwei12 已提交
3515
        self._is_chief = False
3516 3517 3518
        # _parameters_on_pservers records all the parameters distributed on parameter servers.
        self._parameters_on_pservers = None
        # _endpoints is a list about parameter servers ip:port, such as ["ip:port","ip:port"]
T
tangwei12 已提交
3519
        self._endpoints = []
3520 3521 3522
        # if current role is parameter server, the _ps_endpoint is its "ip:port"
        self._ps_endpoint = None
        # trainers_endpoints, it is used for distribution.
3523
        self._trainers_endpoints = []
3524
        # the distributed lookup table names
T
tangwei12 已提交
3525
        self._distributed_lookup_table = None
3526 3527 3528

        # use Deep gradient comrepssion or not
        self._enable_dgc = False
3529 3530
        self._use_lamb = False

3531 3532 3533
        self._nccl_comm_num = 1
        self._use_hierarchical_allreduce = False
        self._hierarchical_allreduce_inter_nranks = 0
3534

3535 3536 3537
        # if this program has been optimized by distributed optimizer
        # fleet_opt will be given a value
        self._fleet_opt = None
D
dongdaxiang 已提交
3538
        self._program_config = None
3539

H
hutuxian 已提交
3540 3541 3542
        # assigned if this program has been parsed by a pipeline optimizer
        self._pipeline_opt = None

3543 3544 3545
        # appending gradients times
        self._appending_grad_times = 0

Y
yuyang18 已提交
3546
    @property
3547
    def _op_role(self):
Y
yuyang18 已提交
3548 3549 3550 3551 3552 3553 3554 3555
        """
        The operator role. In a enum {Forward, Backward, Optimize}.

        Notes: this is a low level API. It is used only for ParallelExecutor to
        duplicate or schedule operator to devices.

        For example, the forward operator should be executed on every device.
        The backward operator should be executed on every device and the
3556
        parameter gradient of backward (use :code:`_op_role_var` to get this
Y
yuyang18 已提交
3557 3558 3559 3560
        variable) operator should be merged to one device. The optimization
        operators should be executed on only one device and broadcast the
        optimization result, i.e., the new parameter, to every other device.
        """
Y
yuyang18 已提交
3561 3562
        return self._current_role

3563 3564
    @_op_role.setter
    def _op_role(self, role):
Y
yuyang18 已提交
3565 3566 3567
        self._current_role = role

    @property
3568
    def _op_role_var(self):
Y
yuyang18 已提交
3569
        """
3570
        The auxiliary variables for :code:`_op_role` property.
Y
yuyang18 已提交
3571

3572
        See Also: :code:`Program._op_role`'s documentation for details.
Y
yuyang18 已提交
3573 3574 3575

        Notes: This is a very low-level API. Users should not use it directly.
        """
3576
        return self.__op_role_var
Y
yuyang18 已提交
3577

3578 3579 3580 3581 3582 3583 3584 3585 3586
    @contextlib.contextmanager
    def _backward_role_guard(self):
        tmp_role = self._current_role

        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Backward
        yield
        self._current_role = tmp_role

S
rename  
sneaxiy 已提交
3587
    @signature_safe_contextmanager
W
Wu Yi 已提交
3588
    def _optimized_guard(self, param_and_grads):
Y
yuyang18 已提交
3589 3590 3591 3592 3593 3594 3595
        """
        A with guard to set :code:`Optimization` :code:`OpRole` and
        :code:`OpRoleVar` automatically.

        Notes: This is a very low level API. Users should not use it directly.

        Args:
3596
            param_and_grads(list): The variables (names) to be optimized.
Y
yuyang18 已提交
3597 3598 3599

        Examples:

3600
            >>> import paddle.fluid as fluid
Y
yuyang18 已提交
3601
            >>> p, g = backward(...)
W
Wu Yi 已提交
3602
            >>> with program._optimized_guard([p,g]):
Y
yuyang18 已提交
3603 3604
            >>>     p = p - 0.001 * g
        """
X
Xin Pan 已提交
3605
        tmp_role = self._current_role
3606
        tmp_var = self.__op_role_var
X
Xin Pan 已提交
3607

Y
yuyang18 已提交
3608 3609
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Optimize
3610
        self.__op_role_var = [
3611 3612 3613
            var.name if isinstance(var, Variable) else var
            for var in param_and_grads
        ]
Y
yuyang18 已提交
3614
        yield
3615
        self.__op_role_var = tmp_var
X
Xin Pan 已提交
3616
        self._current_role = tmp_role
Y
Yu Yang 已提交
3617

S
rename  
sneaxiy 已提交
3618
    @signature_safe_contextmanager
X
Xin Pan 已提交
3619
    def _lr_schedule_guard(self, is_with_opt=False):
3620 3621 3622 3623 3624 3625 3626
        """
        A with guard to set :code:`LRSched` :code:`OpRole` and
        :code:`OpRoleVar` automatically. The :code:`OpRoleVar` is
        set to the target learning rate.

        Notes: This is a very low level API. Users should not use it directly.

X
Xin Pan 已提交
3627 3628 3629 3630
        Args:
            is_with_opt: Only set to true if these ops a in the middle
                 of a bunch of optimize ops so that it can be treated
                 correctly. For example, sgd->lr_op->sgd->lr_op->sgd.
3631 3632 3633

        Examples:

3634
            >>> import paddle.fluid as fluid
3635 3636 3637 3638
            >>> p, g = backward(...)
            >>> with program.lr_schedule_guard():
            >>>     lr = lr * decay
        """
3639 3640

        tmp_role = self._current_role
3641
        tmp_var = self.__op_role_var
3642

3643 3644
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.LRSched
X
Xin Pan 已提交
3645 3646
        if is_with_opt:
            self._current_role = int(OpRole.LRSched) | int(OpRole.Optimize)
3647
        # TODO(typhoonzero): how to set target learning rate var
3648
        self.__op_role_var = []
3649
        yield
3650
        self.__op_role_var = tmp_var
3651
        self._current_role = tmp_role
3652

3653
    def __str__(self):
Y
yuyang18 已提交
3654 3655 3656 3657 3658 3659 3660 3661 3662
        """
        Get the protobuf debug string of this Program.

        Returns:
            (str): The protobuf debug string.

        Raises:
            ValueError: If any of required fields is not set.
        """
Y
Yang Yang(Tony) 已提交
3663 3664
        return self.to_string(True)

F
fengjiayi 已提交
3665 3666 3667
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
Y
yuyang18 已提交
3668

J
Jiabin Yang 已提交
3669 3670 3671
        Args:

            throw_on_error (bool): raise Value error when any of required fields is not set.
F
fengjiayi 已提交
3672

J
Jiabin Yang 已提交
3673
            with_details (bool): True if more details about variables and parameters, e.g., :code:`trainable`, :code:`optimize_attr`, need to print.
Y
yuyang18 已提交
3674

H
haowang101779990 已提交
3675
        Returns:
J
Jiabin Yang 已提交
3676
            str: The debug string describe current Program.
Y
yuyang18 已提交
3677 3678

        Raises:
J
Jiabin Yang 已提交
3679
            ValueError: If any of required fields is not set and throw_on_error is True.
F
fengjiayi 已提交
3680

3681 3682 3683 3684 3685 3686 3687
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                prog_string = prog.to_string(throw_on_error=True, with_details=False)
J
Jiabin Yang 已提交
3688 3689 3690
                print("program string without detial: {}".format(prog_string))
                prog_string_with_detail = prog.to_string(throw_on_error=True, with_details=True)
                print("program string with detial: {}".format(prog_string_with_detail))
F
fengjiayi 已提交
3691 3692 3693 3694 3695 3696 3697 3698 3699
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = ""
            for block in self.blocks:
                res_str += block.to_string(throw_on_error, with_details)
        else:
            protostr = self.desc.serialize_to_string()
3700 3701
            proto = framework_pb2.ProgramDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
3702 3703
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
3704

W
Wu Yi 已提交
3705
    def _get_desc(self):
Y
yuyang18 已提交
3706 3707 3708 3709 3710 3711 3712
        """
        Get the C++ side of `ProgramDesc` object pointer. The C++ object is
        exposed by :code:`pybind`.

        Notes: This is a very low level API. Users should not use this API
        directly.
        """
3713 3714
        return self.desc

X
version  
Xin Pan 已提交
3715 3716 3717
    def _version(self):
        return self.desc._version()

3718
    @dygraph_not_support
3719
    def clone(self, for_test=False):
Y
yuyang18 已提交
3720
        """
3721
        **Notes**:
J
Jiabin Yang 已提交
3722 3723 3724 3725
            **1.** :code:`Program.clone()` **method DOES NOT clone** :ref:`api_fluid_io_DataLoader` .

            **2. Recommend you to use** :code:`clone` **before using** :code:`Opimizer.minimize`.

3726
            **3. This API has no effect in Dygraph Mode**
Y
yuyang18 已提交
3727

3728 3729
        Create a new Program with forward content of original one when ``for_test=True``.
        Create a new Program as the same as original one when ``for_test=False``
3730

3731

J
Jiabin Yang 已提交
3732
        Some operators, e.g., :ref:`api_fluid_layers_batch_norm` , behave differently between
Y
yuyang18 已提交
3733 3734 3735
        training and testing. They have an attribute, :code:`is_test`, to
        control this behaviour. This method will change the :code:`is_test`
        attribute of them to :code:`True` when :code:`for_test=True`.
3736

Y
yuyang18 已提交
3737
        * Set for_test to False when we want to clone the program for training.
3738
        * Set for_test to True when we want to clone the program for testing.
3739 3740
          We will prune the backward and optimize part of the program when you
          use :code:`clone` after :code:`Opimizer.minimize`, but we still
J
Jiabin Yang 已提交
3741
          recommend you to use :code:`clone` before using :code:`Opimizer.minimize`.
Y
yuyang18 已提交
3742

J
Jiabin Yang 已提交
3743 3744
        For Example:
            .. code-block:: python
L
Luo Tao 已提交
3745

J
Jiabin Yang 已提交
3746 3747 3748 3749
                test_program = fluid.default_main_program().clone(for_test=True)
                # Here we use clone before Momentum
                optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
                optimizer.minimize()
3750

J
Jiabin Yang 已提交
3751
        Args:
3752

J
Jiabin Yang 已提交
3753
            for_test (bool): True if change the :code:`is_test` attribute of operators to :code:`True`.
3754

J
Jiabin Yang 已提交
3755 3756
        Returns:
            Program: A new Program with forward content of original one when ``for_test=True``.  A new Program as the same as original one when ``for_test=False``
3757

Y
yuyang18 已提交
3758 3759 3760

        Examples:

J
Jiabin Yang 已提交
3761
        **Notes: The Program's order maybe different after** :code:`clone` **and
3762
        this will not affect your training or testing progress. In the following
J
Jiabin Yang 已提交
3763
        example we give you an simple method** :code:`print_prog(program)` **to
3764
        print Program Descs inorder to make sure you have same print result
J
Jiabin Yang 已提交
3765
        after** :code:`clone`:
3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802
            .. code-block:: python

                import paddle.fluid as fluid
                import six


                def print_prog(prog):
                    for name, value in sorted(six.iteritems(prog.block(0).vars)):
                        print(value)
                    for op in prog.block(0).ops:
                        print("op type is {}".format(op.type))
                        print("op inputs are {}".format(op.input_arg_names))
                        print("op outputs are {}".format(op.output_arg_names))
                        for key, value in sorted(six.iteritems(op.all_attrs())):
                            if key not in ['op_callstack', 'op_role_var']:
                                print(" [ attrs: {}:   {} ]".format(key, value))


        1. To clone a test program, the sample code is:
                .. code-block:: python

                    import paddle.fluid as fluid
                    import six

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))

                    train_program = fluid.Program()
                    startup_program = fluid.Program()
J
Jiabin Yang 已提交
3803 3804 3805

                    # startup_program is used to do some parameter init work,
                    # and main program is used to hold the network
3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
                    with fluid.program_guard(train_program, startup_program):
                        with fluid.unique_name.guard():
                            img = fluid.layers.data(name='image', shape=[784])
                            hidden = fluid.layers.fc(input=img, size=200, act='relu')
                            hidden = fluid.layers.dropout(hidden, dropout_prob=0.5)
                            loss = fluid.layers.cross_entropy(
                                                      input=fluid.layers.fc(hidden, size=10, act='softmax'),
                                        label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
                            avg_loss = fluid.layers.mean(loss)
                            test_program = train_program.clone(for_test=False)
                    print_prog(test_program)
J
Jiabin Yang 已提交
3817 3818 3819 3820 3821 3822 3823 3824 3825

                    # Due to parameter sharing usage for train and test, so we need to use startup program of train
                    # instead of using test startup program, while nothing is in test's startup program

                    # In Paddle Fluid we will share weights by using the same Variable name. In train and test program
                    # all parameters will have the same name and this can make train and test program sharing parameters,
                    # that's why we need to use startup program of train. And for startup program of test, it has nothing,
                    # since it is a new program.

3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872
                    with fluid.program_guard(train_program, startup_program):
                        with fluid.unique_name.guard():
                            sgd = fluid.optimizer.SGD(learning_rate=1e-3)
                            sgd.minimize(avg_loss)


        2. The clone method can be avoid if you create program for training and program for testing individually.
                .. code-block:: python

                    import paddle.fluid as fluid
                    import six

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))
                    def network(is_test):
                        img = fluid.layers.data(name='image', shape=[784])
                        hidden = fluid.layers.fc(input=img, size=200, act='relu')
                        hidden = fluid.layers.dropout(hidden, dropout_prob=0.5)
                        loss = fluid.layers.cross_entropy(
                            input=fluid.layers.fc(hidden, size=10, act='softmax'),
                            label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
                        avg_loss = fluid.layers.mean(loss)
                        return avg_loss


                    train_program_2 = fluid.Program()
                    startup_program_2 = fluid.Program()
                    test_program_2 = fluid.Program()
                    with fluid.program_guard(train_program_2, startup_program_2):
                        with fluid.unique_name.guard():
                             sgd = fluid.optimizer.SGD(learning_rate=1e-3)
                             sgd.minimize(avg_loss)
                    # the test startup program is not used.
                    with fluid.program_guard(test_program_2, fluid.Program()):
                        with fluid.unique_name.guard():
                            loss = network(is_test=True)
                    print(test_program_2)

        The two code snippets above will generate and print same programs.
3873 3874
        """
        if for_test:
3875
            if self._appending_grad_times > 0:
3876 3877 3878 3879 3880 3881 3882
                forward_prog = Program()
                forward_prog.desc = core.prune_backward(self.desc)
                forward_prog.blocks = [
                    Block(forward_prog, i)
                    for i in six.moves.range(forward_prog.desc.num_blocks())
                ]
                forward_prog._sync_with_cpp()
3883 3884 3885
                p = forward_prog._inference_optimize(prune_read_op=False)
            else:
                p = self._inference_optimize(prune_read_op=False)
3886
        else:
3887
            p = Program()
G
gongweibao 已提交
3888 3889
            p.current_block_idx = self.current_block_idx
            p._seed = self._seed
3890
            p.desc = core.ProgramDesc(self.desc)
M
minqiyang 已提交
3891 3892 3893
            p.blocks = [
                Block(p, i) for i in six.moves.range(self.desc.num_blocks())
            ]
G
gongweibao 已提交
3894 3895

            p._current_role = self._current_role
3896
            p.__op_role_var = self.__op_role_var
3897
            p._appending_grad_times = self._appending_grad_times
G
gongweibao 已提交
3898

W
Wu Yi 已提交
3899
            p._sync_with_cpp()
3900

W
Wu Yi 已提交
3901
        p._copy_param_info_from(self)
W
Wu Yi 已提交
3902
        p._copy_data_info_from(self)
3903
        p._copy_dist_param_info_from(self)
Y
Yu Yang 已提交
3904
        return p
3905

3906
    def _prune(self, targets):
Y
yuyang18 已提交
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919
        """
        Prune operators and variables which are not needed to generate
        :code:`targets`.

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            targets(list|Variable|Operator): A list of variables or operators
                need to be pruned

        Returns:
            Program:  A new, pruned program.
3920 3921 3922 3923
        """

        if not isinstance(targets, list):
            targets = [targets]
Y
yuyang18 已提交
3924

3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
                    # After transpiler processing, the op that output this
                    # variable maybe has been changed, so t.op is not reliable
                    # and we need to find the current op that generate this
                    # variable here.
                    t.op = None
                    global_block = self.global_block()
                    for idx, op in enumerate(global_block.ops):
                        if t.name in op.output_arg_names:
                            t.op = op
                            break

                    t = t.op
                    if t is None:
                        raise ValueError(
                            "The target variable must have an "
                            "associated operator that generates it.")
                else:
                    raise ValueError("All targets of prune() can only be "
                                     "Variable or Operator.")

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
        res.desc = core.prune(self.desc, set(), targets_idx)
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
        res._sync_with_cpp()
        return res

    def _prune_with_input(self, feeded_var_names, targets):
Y
yuyang18 已提交
3959
        """
3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976
        Prune operators and variables which are not needed to generate
        :code:`targets`. Prune operators and variables which are needed 
        to generate feeded_var 

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            feeded_var_names(list|str): A list of variable names from where
                pruning start. If it is set as [], this API works just like _prune()
            targets(list|Variable|Operator): A list of variables or operators
                need to be pruned

        Returns:
            Program:  A new, pruned program.
        """

3977 3978
        if not isinstance(feeded_var_names, list):
            feeded_var_names = [feeded_var_names]
3979 3980
        if not isinstance(targets, list):
            targets = [targets]
3981 3982 3983 3984 3985 3986

        for var in feeded_var_names:
            if not isinstance(var, six.string_types):
                raise ValueError("All feeded_var_names of prune() can only be "
                                 "str.")

3987 3988 3989 3990
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
3991 3992
                    # After transpiler processing, the op that output this
                    # variable maybe has been changed, so t.op is not reliable
3993
                    # and we need to find the current op that generate this
3994 3995 3996 3997 3998 3999 4000 4001
                    # variable here.
                    t.op = None
                    global_block = self.global_block()
                    for idx, op in enumerate(global_block.ops):
                        if t.name in op.output_arg_names:
                            t.op = op
                            break

4002
                    t = t.op
4003 4004 4005 4006
                    if t is None:
                        raise ValueError(
                            "The target variable must have an "
                            "associated operator that generates it.")
4007
                else:
4008 4009
                    raise ValueError("All targets of prune() can only be "
                                     "Variable or Operator.")
4010 4011 4012

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
4013
        res.desc = core.prune(self.desc, set(feeded_var_names), targets_idx)
M
minqiyang 已提交
4014 4015 4016
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
4017
        res._sync_with_cpp()
4018 4019
        return res

X
Xin Pan 已提交
4020
    def _inference_optimize(self, prune_read_op=True):
Y
yuyang18 已提交
4021
        """
F
fengjiayi 已提交
4022 4023 4024 4025 4026
        This method will create a new program and do following adjustments on it:
        1. Remove all reader variables and their creator ops if exist.

        2. Remove the :code:`read_op` if exists.

4027
        3. change the :code:`is_test`
Y
yuyang18 已提交
4028 4029 4030
        attribute of operators to :code:`True`. All the :code:`Parameter`
        information will be lost.

4031
        Args:
X
Xin Pan 已提交
4032 4033
            prune_read_op(bool): remove the read ops that are added by py_reader
                                 for cpp inference library
4034

Y
yuyang18 已提交
4035 4036 4037 4038 4039 4040
        Notes: This API is a very low level API. Use
        :code:`Program.clone(for_test=True)` instead.

        Returns:
            Program: The new program.
        """
4041
        res = Program()
4042
        res.desc = core.ProgramDesc(self.desc)
F
fengjiayi 已提交
4043 4044 4045 4046

        # remove all readers and the read_op if exist
        read_op_idx = 0
        root_block = res.desc.block(0)
X
Xin Pan 已提交
4047
        if prune_read_op:
4048 4049 4050 4051 4052 4053 4054 4055 4056
            while True:
                if read_op_idx >= root_block.op_size() or root_block.op(
                        read_op_idx).type() == 'read':
                    break
                read_op_idx += 1
            if read_op_idx < root_block.op_size():
                root_block._remove_op(0, read_op_idx + 1)
            for var in root_block.all_vars():
                if var.type() == core.VarDesc.VarType.READER:
M
minqiyang 已提交
4057
                    root_block._remove_var(cpt.to_bytes(var.name()))
F
fengjiayi 已提交
4058 4059

        # change all `is_test` attributes to True
M
minqiyang 已提交
4060
        for i in six.moves.range(res.desc.num_blocks()):
4061
            block = res.desc.block(i)
M
minqiyang 已提交
4062
            for j in six.moves.range(block.op_size()):
4063 4064
                op = block.op(j)
                if op.has_attr('is_test'):
W
Wu Yi 已提交
4065
                    op._set_attr('is_test', True)
M
minqiyang 已提交
4066 4067 4068
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
4069
        res._sync_with_cpp()
4070 4071
        return res

4072 4073
    @staticmethod
    def parse_from_string(binary_str):
Y
yuyang18 已提交
4074
        """
J
Jiabin Yang 已提交
4075 4076 4077 4078
        **Notes**:
            **1. All information about parameters will be lost after serialization**

            **2. This API has no effect in Dygraph mode**
Y
yuyang18 已提交
4079

4080 4081
        Deserialize a Program from  `protobuf <https://en.wikipedia.org/wiki/Protocol_Buffers>`_  binary string.
        This method always use to save and load model
Y
yuyang18 已提交
4082

J
Jiabin Yang 已提交
4083
        Args:
Y
yuyang18 已提交
4084

J
Jiabin Yang 已提交
4085
            binary_str_type (str): the binary prootbuf string.
4086

J
Jiabin Yang 已提交
4087 4088
        Returns:
            Program: A deserialized Program.
4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                startup_prog = fluid.Program()
                main_prog = fluid.Program()
                with fluid.program_guard(startup_prog, main_prog):
                    x = fluid.layers.data(
                        name='X', shape=[1000, 784], dtype='float32', append_batch_size=False)

                    y = fluid.layers.data(
                        name='Y', shape=[784, 100], dtype='float32', append_batch_size=False)

                    z = fluid.layers.mul(x=x, y=y)

                    binary_str = fluid.default_main_program().desc.serialize_to_string()
                    prog_restored = fluid.default_main_program().parse_from_string(binary_str)

                    print(fluid.default_main_program())
                    print(prog_restored)
Y
yuyang18 已提交
4111
        """
4112 4113
        p = Program()
        p.desc = core.ProgramDesc(binary_str)
M
minqiyang 已提交
4114
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
W
Wu Yi 已提交
4115
        p._sync_with_cpp()
4116
        return p
Y
Yu Yang 已提交
4117

4118
    @staticmethod
4119
    def _construct_from_desc(desc):
4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134
        """
        Construct a program from program desc.

        Args:
            desc(core.ProgramDesc): The program desc for constructing.

        Returns:
            Program: A program.
        """
        p = Program()
        p.desc = desc
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
        p._sync_with_cpp()
        return p

D
dzhwinter 已提交
4135 4136
    @property
    def random_seed(self):
Y
yuyang18 已提交
4137
        """
J
Jiabin Yang 已提交
4138
        The default random seed for random operators in Program. ``0`` means get
Y
yuyang18 已提交
4139 4140
        the random seed from random device.

J
Jiabin Yang 已提交
4141 4142 4143 4144
        **Notes: It must be set before the operators have been added.**

        Returns:
            int64: Random seed in current Program
4145

4146 4147 4148 4149 4150 4151 4152 4153

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                random_seed = prog.random_seed
4154 4155 4156
                x_var = fluid.layers.data(name="X", shape=[3,3], dtype="float32", append_batch_size=False)

                # Here we need to set random seed before we use fluid.layers.dropout
4157 4158
                print(random_seed)
                prog.random_seed = 1
4159 4160
                z_var = fluid.layers.dropout(x_var, 0.7)

4161
                print(prog.random_seed)
Y
yuyang18 已提交
4162
        """
D
dzhwinter 已提交
4163 4164
        return self._seed

Q
qiaolongfei 已提交
4165 4166
    @property
    def num_blocks(self):
Y
yuyang18 已提交
4167
        """
4168 4169
        The number of :ref:`api_guide_Block_en`  in this Program.

J
Jiabin Yang 已提交
4170 4171 4172 4173
        **Notes: This API has no effect in Dygraph mode**

        Returns:
            int(Platform-dependent size): num of :ref:`api_guide_Block_en`  in current Program
4174

4175 4176 4177 4178 4179 4180 4181 4182 4183

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                num_blocks = prog.num_blocks
                print(num_blocks)
4184 4185


Y
yuyang18 已提交
4186
        """
Q
qiaolongfei 已提交
4187 4188
        return self.desc.num_blocks()

D
dzhwinter 已提交
4189 4190 4191 4192 4193 4194
    @random_seed.setter
    def random_seed(self, seed):
        if not isinstance(seed, int):
            raise ValueError("Seed must be a integer.")
        self._seed = seed

Y
Yu Yang 已提交
4195
    def __repr__(self):
4196
        return self.__str__()
4197

Y
Yu Yang 已提交
4198
    def global_block(self):
Y
yuyang18 已提交
4199
        """
J
Jiabin Yang 已提交
4200 4201
        **Notes**:
            **This API has no effect in Dygraph mode**
4202 4203 4204

        Get the first :ref:`api_guide_Block_en` of this Program.

J
Jiabin Yang 已提交
4205 4206
        Returns:
            :ref:`api_guide_Block_en`: The first :ref:`api_guide_Block_en`  of this Program.
4207

4208 4209 4210 4211 4212 4213 4214 4215 4216

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                gb_block = prog.global_block()
                print(gb_block)
4217

Y
yuyang18 已提交
4218
        """
Y
Yu Yang 已提交
4219 4220
        return self.blocks[0]

Q
Qiao Longfei 已提交
4221
    def block(self, index):
Y
yuyang18 已提交
4222
        """
J
Jiabin Yang 已提交
4223 4224
        **Notes**:
            **This API has no effect in Dygraph mode**
Y
yuyang18 已提交
4225

4226 4227
        Get the :code:`index`  :ref:`api_guide_Block_en`  of this Program

J
Jiabin Yang 已提交
4228 4229
        Args:
            index (int) - The index of  :ref:`api_guide_Block_en`  to get
4230

J
Jiabin Yang 已提交
4231 4232
        Returns:
            :ref:`api_guide_Block_en`: The :code:`index` block
4233 4234 4235 4236 4237 4238 4239 4240 4241

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                block_0 = prog.block(0)
                print(block_0)
Y
yuyang18 已提交
4242
        """
Q
Qiao Longfei 已提交
4243 4244
        return self.blocks[index]

Y
Yu Yang 已提交
4245
    def current_block(self):
Y
yuyang18 已提交
4246
        """
J
Jiabin Yang 已提交
4247 4248
        **Notes**:
            **This API has no effect in Dygraph mode**
4249

J
Jiabin Yang 已提交
4250 4251
        Get the current  :ref:`api_guide_Block_en` . The :code:`current`  :ref:`api_guide_Block_en`
        is the  :ref:`api_guide_Block_en`  to append operators.
4252

J
Jiabin Yang 已提交
4253 4254
        Returns:
             :ref:`api_guide_Block_en`: The :code:`index`  :ref:`api_guide_Block_en`
4255

4256 4257 4258 4259 4260 4261 4262 4263
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                current_blk = prog.current_block()
                print(current_blk)
Y
yuyang18 已提交
4264
        """
Y
Yu Yang 已提交
4265 4266
        return self.blocks[self.current_block_idx]

W
Wu Yi 已提交
4267
    def _create_block(self, parent_idx=None):
Y
yuyang18 已提交
4268 4269 4270 4271 4272
        """
        Create a new block with the :code:`parent_idx` and change the current block
        to new block.

        Args:
J
Jiabin Yang 已提交
4273

Y
yuyang18 已提交
4274 4275 4276 4277 4278
            parent_idx(int): The parent block index.

        Returns:
            Block: The new block.
        """
Y
Yu Yang 已提交
4279
        new_block_idx = len(self.blocks)
F
update  
fengjiayi 已提交
4280 4281 4282
        parent = self.current_block() if parent_idx is None else self.block(
            parent_idx)
        self.desc.append_block(parent.desc)
Y
Yu Yang 已提交
4283 4284 4285 4286
        self.current_block_idx = new_block_idx
        self.blocks.append(Block(self, self.current_block_idx))
        return self.current_block()

W
Wu Yi 已提交
4287
    def _rollback(self):
Y
yuyang18 已提交
4288 4289 4290 4291 4292
        """
        Exit a code block, i.e., roll back to the parent block.
        Returns:
            None
        """
Y
Yu Yang 已提交
4293 4294
        self.current_block_idx = self.current_block().parent_idx

W
Wu Yi 已提交
4295
    def _sync_with_cpp(self):
Y
yuyang18 已提交
4296 4297 4298 4299 4300 4301 4302 4303 4304 4305
        """
        Synchronize Python instance to its binding C++ object instance.
        If the program is modified in C++ space, this method should be invoked.

        Notes: This is a very low level API. Users should not invoke it
        directly.

        Returns:
            None
        """
Q
Qiao Longfei 已提交
4306 4307 4308
        for block_idx in range(len(self.blocks), self.desc.num_blocks()):
            self.blocks.append(Block(self, block_idx))
        for block in self.blocks:
W
Wu Yi 已提交
4309
            block._sync_with_cpp()
Q
Qiao Longfei 已提交
4310

W
Wu Yi 已提交
4311
    def _copy_param_info_from(self, other):
4312
        """
4313
        Copy the information of parameters from other program.
D
dzhwinter 已提交
4314

Y
yuyang18 已提交
4315 4316 4317
        Notes: This is a very low level API. Users should not invoke it
        directly.

4318 4319 4320 4321 4322 4323 4324
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
W
Wu Yi 已提交
4325
            raise TypeError("_copy_param_info_from should be invoked with "
4326 4327 4328
                            "Program")

        if len(self.blocks) != len(other.blocks):
W
Wu Yi 已提交
4329
            raise ValueError("_copy_param_info_from should be invoked with two "
4330
                             "program, with represent the same topology")
W
Wu Yi 已提交
4331
        self.global_block()._copy_param_info_from(other.global_block())
4332

4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347
    def _copy_dist_param_info_from(self, other):
        """
        Copy the information of distributed information from other program.

        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
            raise TypeError("_copy_dist_param_info_from should be invoked with "
                            "Program")
        self._is_distributed = other._is_distributed
        self._is_chief = other._is_chief
4348
        self._parameters_on_pservers = other._parameters_on_pservers
4349
        self._endpoints = other._endpoints
4350
        self._ps_endpoint = other._ps_endpoint
4351 4352
        self._distributed_lookup_table = other._distributed_lookup_table

W
Wu Yi 已提交
4353
    def _copy_data_info_from(self, other):
F
fengjiayi 已提交
4354 4355
        """
        Copy the information of data variables from other program.
D
dzhwinter 已提交
4356

Y
yuyang18 已提交
4357 4358 4359
        Notes: This is a very low level API. Users should not invoke it
        directly.

F
fengjiayi 已提交
4360 4361 4362 4363 4364 4365 4366
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
W
Wu Yi 已提交
4367
            raise TypeError("_copy_param_info_from should be invoked with "
F
fengjiayi 已提交
4368 4369 4370
                            "Program")

        if len(self.blocks) != len(other.blocks):
W
Wu Yi 已提交
4371
            raise ValueError("_copy_param_info_from should be invoked with two "
F
fengjiayi 已提交
4372
                             "program, with represent the same topology")
4373
        for var in list(other.global_block().vars.values()):
F
fengjiayi 已提交
4374 4375
            if var.is_data:
                self.global_block().var(var.name).is_data = True
H
Huihuang Zheng 已提交
4376 4377
            if var.desc.need_check_feed():
                self.global_block().var(var.name).desc.set_need_check_feed(True)
F
fengjiayi 已提交
4378

4379
    @dygraph_not_support
4380
    def list_vars(self):
Y
yuyang18 已提交
4381
        """
J
Jiabin Yang 已提交
4382
        Get all :ref:`api_guide_Variable_en` from this Program. A iterable object is returned.
Y
yuyang18 已提交
4383

J
Jiabin Yang 已提交
4384 4385
        Returns:
            iterable :ref:`api_guide_Variable_en`: The Generator will yield every variable in this program.
4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                img = fluid.layers.data(name='img', shape=[1,28,28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[128,1], dtype='int64')
                for var in prog.list_vars():
                    print(var)
Y
yuyang18 已提交
4397
        """
4398
        for each_block in self.blocks:
4399
            for each_var in list(each_block.vars.values()):
4400 4401
                yield each_var

Y
Yu Yang 已提交
4402

Y
Yu Yang 已提交
4403
class Parameter(Variable):
4404
    """
4405
    Parameter is derived from Variable. A parameter is a persistable
4406
    Variable, and will be updated by optimizers after each iteration.
4407
    The training of a neural network is essentially the updating of
4408 4409
    its parameters.

4410
    Relative to a general Variable, a Parameter has several its own
4411 4412
    member variables:

4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424
    Args:
        trainable(bool): True if the parameter need to be updated after
            iterations.
        optimize_attr(map): Parameter attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the parameter. Default: None
        gradient_clip_attr(BaseGradientClipAttr): The gradint clip strategy
            which will be applied on the parameter. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this parameter.
4425 4426
    """

Y
Yu Yang 已提交
4427
    def __init__(self, block, shape, dtype, **kwargs):
4428 4429 4430 4431 4432
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

Y
Yu Yang 已提交
4433
        if len(shape) == 0:
4434 4435
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")
Y
Yu Yang 已提交
4436 4437 4438

        for each in shape:
            if each < 0:
4439 4440 4441
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))
4442 4443 4444

        Variable.__init__(
            self, block, persistable=True, shape=shape, dtype=dtype, **kwargs)
Y
Yu Yang 已提交
4445 4446 4447 4448
        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

4449 4450
        self.regularizer = kwargs.get('regularizer', None)

F
fengjiayi 已提交
4451
        self.gradient_clip_attr = kwargs.get('gradient_clip_attr', None)
Y
Yu Yang 已提交
4452

W
wanghaoshuang 已提交
4453
        self.do_model_average = kwargs.get('do_model_average', None)
W
wanghaoshuang 已提交
4454

4455 4456
        self.is_distributed = False

F
fengjiayi 已提交
4457 4458 4459
    def __str__(self):
        return self.to_string(True)

F
update  
fengjiayi 已提交
4460 4461 4462
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
D
dzhwinter 已提交
4463

F
update  
fengjiayi 已提交
4464 4465 4466 4467 4468 4469 4470 4471
        Args:
            throw_on_error(bool): raise exception when self is not initialized
                when throw_on_error is True
            with_details(bool): more details about variables and parameters
                (e.g. trainable, optimize_attr, ...) will be printed when with_details is True

        Returns(str): The debug string.

4472 4473 4474 4475 4476 4477 4478 4479 4480
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                rlt = fluid.layers.data("fake_data", shape=[1,1], dtype='float32')
                debug_str = prog.to_string(throw_on_error=True, with_details=False)
                print(debug_str)
F
update  
fengjiayi 已提交
4481 4482 4483 4484 4485 4486
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = Variable.to_string(self, throw_on_error, True)
            additional_attr = ("trainable", "optimize_attr", "regularizer",
W
wanghaoshuang 已提交
4487
                               "gradient_clip_attr", "do_model_average")
F
update  
fengjiayi 已提交
4488
            for attr_name in additional_attr:
4489 4490
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))
F
update  
fengjiayi 已提交
4491 4492
        else:
            res_str = Variable.to_string(self, throw_on_error, False)
F
fengjiayi 已提交
4493 4494 4495 4496
        return res_str

    __repr__ = __str__

Y
Yu Yang 已提交
4497

Y
Yu Yang 已提交
4498
# program is a global instance.
Y
Yu Yang 已提交
4499 4500
_main_program_ = Program()
_startup_program_ = Program()
4501

4502

4503
def default_startup_program():
Y
Yu Yang 已提交
4504
    """
Y
yuyang18 已提交
4505 4506
    Get default/global startup program.

J
Jiabin Yang 已提交
4507 4508 4509
    The layer function in :ref:`api_fluid_layers` will create parameters, :ref:`api_paddle_data_reader_reader` ,
    `NCCL <https://developer.nvidia.com/nccl>`_ handles as global variables. The :code:`startup_program` will
    initialize them by the OPs in startup  :ref:`api_fluid_Program` . The  :ref:`api_fluid_layers`  function will
Y
yuyang18 已提交
4510 4511 4512
    append these initialization operators into startup program.

    This method will return the :code:`default` or the :code:`current` startup
J
Jiabin Yang 已提交
4513
    program. Users can use  :ref:`api_fluid_program_guard`  to switch :ref:`api_fluid_Program` .
4514

J
Jiabin Yang 已提交
4515
    Returns: current default startup :ref:`api_fluid_Program`
4516

J
Jiabin Yang 已提交
4517
    Returns type: :ref:`api_fluid_Program`
4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid

            main_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(main_program=main_program, startup_program=startup_program):
                x = fluid.layers.data(name="x", shape=[-1, 784], dtype='float32')
                y = fluid.layers.data(name="y", shape=[-1, 1], dtype='int32')
                z = fluid.layers.fc(name="fc", input=x, size=10, act="relu")

                print("main program is: {}".format(fluid.default_main_program()))
                print("start up program is: {}".format(fluid.default_startup_program()))
Y
Yu Yang 已提交
4533
    """
Y
Yu Yang 已提交
4534
    return _startup_program_
4535

4536

4537
def default_main_program():
Y
Yu Yang 已提交
4538
    """
4539 4540 4541 4542 4543
    This API can be used to get ``default main program`` which store the 
    descriptions of ``op`` and ``variable``.
    
    For example ``z = fluid.layers.elementwise_add(x, y)`` will create a new ``elementwise_add`` 
    ``op`` and a new ``z`` ``variable``, and they will be recorded in ``default main program`` 
Y
yuyang18 已提交
4544

4545 4546
    The ``default_main_program`` is the default value for ``Program`` parameter in 
    a lot of ``fluid`` APIs. For example, the :code:`Executor.run()` will execute the
Y
yuyang18 已提交
4547
    :code:`default_main_program` when the program is not specified.
4548

4549 4550
    If you want to replace the ``default main program``, you can use :ref:`api_fluid_program_guard`
    
Y
Yu Yang 已提交
4551
    Returns:
4552
        :ref:`api_fluid_Program`: a ``Program`` which holding the descriptions of ops and variables in the network.
4553 4554 4555 4556 4557

    Examples:
        ..  code-block:: python

            import paddle.fluid as fluid
4558

4559
            # Sample Network:
4560 4561
            data = fluid.data(name='image', shape=[None, 3, 224, 224], dtype='float32')
            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580
            
            conv1 = fluid.layers.conv2d(data, 4, 5, 1, act=None)
            bn1 = fluid.layers.batch_norm(conv1, act='relu')
            pool1 = fluid.layers.pool2d(bn1, 2, 'max', 2)
            conv2 = fluid.layers.conv2d(pool1, 16, 5, 1, act=None)
            bn2 = fluid.layers.batch_norm(conv2, act='relu')
            pool2 = fluid.layers.pool2d(bn2, 2, 'max', 2)
            
            fc1 = fluid.layers.fc(pool2, size=50, act='relu')
            fc2 = fluid.layers.fc(fc1, size=102, act='softmax')
            
            loss = fluid.layers.cross_entropy(input=fc2, label=label)
            loss = fluid.layers.mean(loss)
            opt = fluid.optimizer.Momentum(
                learning_rate=0.1,
                momentum=0.9,
                regularization=fluid.regularizer.L2Decay(1e-4))
            opt.minimize(loss)
            
4581
            #print the number of blocks in the program, 1 in this case
4582
            print(fluid.default_main_program().num_blocks)
4583 4584

            #print the description of variable 'image'
4585
            print(fluid.default_main_program().blocks[0].var('image'))
4586

Y
Yu Yang 已提交
4587
    """
Y
Yu Yang 已提交
4588
    return _main_program_
Y
Yu Yang 已提交
4589 4590 4591 4592 4593


def switch_main_program(program):
    """
    Switch the main program to a new program.
4594

Y
Yu Yang 已提交
4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608
    Args:
        program(Program): The new main program

    Returns:
        Program: The previous main program
    """
    global _main_program_
    prev_program = _main_program_
    _main_program_ = program
    return prev_program


def switch_startup_program(program):
    """
4609
    Switch the startup program to a new program
Y
Yu Yang 已提交
4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
    Args:
        program(Program): The new startup program

    Returns:
        Program: The previous startup program
    """
    global _startup_program_
    prev_program = _startup_program_
    _startup_program_ = program
    return prev_program


S
rename  
sneaxiy 已提交
4622
@signature_safe_contextmanager
Y
Yu Yang 已提交
4623 4624
def program_guard(main_program, startup_program=None):
    """
4625 4626
    Change the global main program and startup program with `"with"` statement.
    Layer functions in the Python `"with"` block will append operators and
Y
yuyang18 已提交
4627
    variables to the new main programs.
4628

G
guofei 已提交
4629 4630 4631 4632 4633 4634 4635
    Args:
        main_program(Program): New main program inside `"with"` statement.
        startup_program(Program, optional): New startup program inside `"with"` 
            statement. :code:`None` means not changing startup program, 
            default_startup_program is still used.
            Default: None.

Y
Yu Yang 已提交
4636
    Examples:
4637 4638 4639
       .. code-block:: python
       
         import paddle.fluid as fluid
Y
yuyang18 已提交
4640

4641 4642 4643
         main_program = fluid.Program()
         startup_program = fluid.Program()
         with fluid.program_guard(main_program, startup_program):
G
guofei 已提交
4644
             data = fluid.data(name='image', shape=[None, 784, 784], dtype='float32')
4645
             hidden = fluid.layers.fc(input=data, size=10, act='relu')
Y
yuyang18 已提交
4646 4647 4648

    Notes: The temporary :code:`Program` can be used if the user does not need
    to construct either of startup program or main program.
4649

Y
Yu Yang 已提交
4650
    Examples:
4651
       .. code-block:: python
Y
yuyang18 已提交
4652

4653 4654 4655 4656 4657
         import paddle.fluid as fluid

         main_program = fluid.Program()
         # does not care about startup program. Just pass a temporary value.
         with fluid.program_guard(main_program, fluid.Program()):
G
guofei 已提交
4658 4659
             data = fluid.data(name='image', shape=[None, 784, 784], dtype='float32')
    
Y
Yu Yang 已提交
4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671
    """
    if not isinstance(main_program, Program):
        raise TypeError("main_program should be Program")
    main_program = switch_main_program(main_program)
    if startup_program is not None:
        if not isinstance(startup_program, Program):
            raise TypeError("startup_program should be Program")
        startup_program = switch_startup_program(startup_program)
    yield
    switch_main_program(main_program)
    if startup_program is not None:
        switch_startup_program(startup_program)
X
xuwei06 已提交
4672 4673


W
Wu Yi 已提交
4674
def _get_var(name, program=None):
X
xuwei06 已提交
4675
    """
Y
yuyang18 已提交
4676
    Get a variable by name from the global block of a program.
F
fengjiayi 已提交
4677

X
xuwei06 已提交
4678 4679 4680
    Args:
        name(str): name of the variable
        program(Program|None): program object.
T
tangwei12 已提交
4681
        If None, default_global_program() will be used.
X
xuwei06 已提交
4682 4683 4684 4685 4686 4687 4688

    Returns:
        Variable
    """
    if program is None:
        program = default_main_program()
    assert isinstance(name, str)
4689
    assert isinstance(program, Program)
X
xuwei06 已提交
4690 4691

    return program.global_block().var(name)
4692 4693


S
rename  
sneaxiy 已提交
4694
@signature_safe_contextmanager
L
lujun 已提交
4695 4696 4697 4698
def _dygraph_guard(tracer):
    global _dygraph_tracer_
    tmp_trace = _dygraph_tracer_
    _dygraph_tracer_ = tracer
M
minqiyang 已提交
4699

4700
    yield
P
Paddle CI 已提交
4701

L
lujun 已提交
4702
    _dygraph_tracer_ = tmp_trace
P
Paddle CI 已提交
4703 4704


S
rename  
sneaxiy 已提交
4705
@signature_safe_contextmanager
L
lujun 已提交
4706 4707 4708 4709
def _dygraph_place_guard(place):
    global _dygraph_current_expected_place_
    tmp_place = _dygraph_current_expected_place_
    _dygraph_current_expected_place_ = place
M
minqiyang 已提交
4710

4711
    yield
M
minqiyang 已提交
4712

L
lujun 已提交
4713
    _dygraph_current_expected_place_ = tmp_place
4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735


def load_op_library(lib_filename):
    """
    Load a dynamic library, including custom operators and kernels.
    When library is loaded, ops and kernels registered in the library
    will be available in PaddlePaddle main process.
    Please note, the type of custom operators cann't have the same type
    with the existing operators in the framework.

    Args:
        lib_filename (str): name of dynamic library.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            #fluid.load_op_library('custom_op.so')

    """
    core.load_op_library(lib_filename)
    OpProtoHolder.instance().update_op_proto()